spb/llmindex Public

1/**2 * llmindex.io — pure aggregation of fitted parameters → scores (NO I/O)3 * Author:  Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * License: Proprietary — © Simon-Pierre Boucher, all rights reserved6 */7import {8  CI_Z,9  CONTAMINATION_DELTA_FLOOR,10  DOMAIN_WEIGHTS,11  SUBMETRIC_WEIGHTS,12  THETA_SCALE,13  type SubMetricKey,14} from './weights';15import { type Domain } from './domains';1617export * from './domains';18export * from './weights';1920export interface ScoreWithCI {21  score: number;22  scoreLow: number;23  scoreHigh: number;24}2526export interface DomainSubMetrics {27  /** 2PL ability θ for the domain (logits). */28  theta: number;29  /** Standard error of θ from the Fisher information. */30  thetaSe: number;31  /** Answer stability across k samples/paraphrases, in [0,1]. */32  consistency?: number | null;33  /** 1 − ECE, in [0,1]. */34  calibration?: number | null;35  /** Fixed-vs-perturbed accuracy gap, in [0,1]. */36  contaminationDelta?: number | null;37}3839const clamp01 = (x: number): number => Math.min(1, Math.max(0, x));40const clampIndex = (x: number): number =>41  Math.min(THETA_SCALE.max, Math.max(THETA_SCALE.min, x));4243/** Logistic squash of θ → [0,1]; P(correct) on a median (b=0, a=1) item. */44export function abilityToUnit(theta: number): number {45  return 1 / (1 + Math.exp(-theta));46}4748/** Rescale θ (± CI from SE) directly to the 0–1000 display scale. */49export function thetaToIndex(theta: number, thetaSe: number): ScoreWithCI {50  const score = clampIndex(THETA_SCALE.center + THETA_SCALE.slope * theta);51  const half = THETA_SCALE.slope * CI_Z * thetaSe;52  return {53    score: Math.round(score),54    scoreLow: Math.round(clampIndex(score - half)),55    scoreHigh: Math.round(clampIndex(score + half)),56  };57}5859export function contaminationResistance(delta: number): number {60  return clamp01(1 - delta / CONTAMINATION_DELTA_FLOOR);61}6263/**64 * Blend available sub-metrics into a domain composite in [0,1], renormalizing65 * weights over the metrics actually present (a missing metric neither rewards66 * nor punishes). accuracy_irt is always required.67 */68export function domainComposite(m: DomainSubMetrics): number {69  const values: Partial<Record<SubMetricKey, number>> = {70    accuracy_irt: abilityToUnit(m.theta),71  };72  if (m.consistency != null) values.consistency = clamp01(m.consistency);73  if (m.calibration != null) values.calibration = clamp01(m.calibration);74  if (m.contaminationDelta != null)75    values.contamination_resistance = contaminationResistance(m.contaminationDelta);7677  let weightSum = 0;78  let acc = 0;79  for (const [key, value] of Object.entries(values) as [SubMetricKey, number][]) {80    const w = SUBMETRIC_WEIGHTS[key];81    weightSum += w;82    acc += w * value;83  }84  return acc / weightSum;85}8687/**88 * Domain composite with CI, propagated from θ SE via the delta method through89 * the accuracy term (the only stochastic fit parameter in the composite).90 */91export function domainScore(m: DomainSubMetrics): ScoreWithCI {92  const composite = domainComposite(m);93  const p = abilityToUnit(m.theta);94  const present: SubMetricKey[] = ['accuracy_irt'];95  if (m.consistency != null) present.push('consistency');96  if (m.calibration != null) present.push('calibration');97  if (m.contaminationDelta != null) present.push('contamination_resistance');98  const weightSum = present.reduce((s, k) => s + SUBMETRIC_WEIGHTS[k], 0);99  const wAcc = SUBMETRIC_WEIGHTS.accuracy_irt / weightSum;100  const half = 1000 * wAcc * p * (1 - p) * CI_Z * m.thetaSe;101  const score = 1000 * composite;102  return {103    score: Math.round(score),104    scoreLow: Math.round(clampIndex(score - half)),105    scoreHigh: Math.round(clampIndex(score + half)),106  };107}108109/**110 * Global Index: weighted average of per-domain composites over the domains a111 * model was actually evaluated on (weights renormalized), rescaled to 0–1000.112 * CI combines domain CI half-widths in quadrature (domains fit independently).113 */114export function globalIndex(perDomain: Partial<Record<Domain, ScoreWithCI>>): ScoreWithCI | null {115  const entries = Object.entries(perDomain) as [Domain, ScoreWithCI][];116  if (entries.length === 0) return null;117  const weightSum = entries.reduce((s, [d]) => s + DOMAIN_WEIGHTS[d], 0);118  let score = 0;119  let varAcc = 0;120  for (const [d, s] of entries) {121    const w = DOMAIN_WEIGHTS[d] / weightSum;122    score += w * s.score;123    const half = (s.scoreHigh - s.scoreLow) / 2;124    varAcc += (w * half) ** 2;125  }126  const half = Math.sqrt(varAcc);127  return {128    score: Math.round(score),129    scoreLow: Math.round(clampIndex(score - half)),130    scoreHigh: Math.round(clampIndex(score + half)),131  };132}133134export interface ParetoPoint {135  slug: string;136  /** Quality score (higher better), e.g. Global Index. */137  score: number;138  /** Cost in USD per 1k items (lower better). */139  costPer1kItems: number;140}141142/** Efficiency frontier: models not dominated on (score↑, cost↓). Never a blended number. */143export function paretoFrontier(points: ParetoPoint[]): ParetoPoint[] {144  const sorted = [...points].sort((a, b) => a.costPer1kItems - b.costPer1kItems || b.score - a.score);145  const frontier: ParetoPoint[] = [];146  let best = -Infinity;147  for (const p of sorted) {148    if (p.score > best) {149      frontier.push(p);150      best = p.score;151    }152  }153  return frontier;154}155156/** Bradley-Terry strength → θ-like scale used by duel domains (log-strength standardized). */157export function btStrengthToTheta(logStrengths: number[]): number[] {158  const mean = logStrengths.reduce((a, b) => a + b, 0) / logStrengths.length;159  const sd =160    Math.sqrt(logStrengths.reduce((a, b) => a + (b - mean) ** 2, 0) / logStrengths.length) || 1;161  return logStrengths.map((s) => (s - mean) / sd);162}163