/** * llmindex.io — pure aggregation of fitted parameters → scores (NO I/O) * Author: Simon-Pierre Boucher * Contact: contact@spboucher.ai * License: Proprietary — © Simon-Pierre Boucher, all rights reserved */ import { CI_Z, CONTAMINATION_DELTA_FLOOR, DOMAIN_WEIGHTS, SUBMETRIC_WEIGHTS, THETA_SCALE, type SubMetricKey, } from './weights'; import { type Domain } from './domains'; export * from './domains'; export * from './weights'; export interface ScoreWithCI { score: number; scoreLow: number; scoreHigh: number; } export interface DomainSubMetrics { /** 2PL ability θ for the domain (logits). */ theta: number; /** Standard error of θ from the Fisher information. */ thetaSe: number; /** Answer stability across k samples/paraphrases, in [0,1]. */ consistency?: number | null; /** 1 − ECE, in [0,1]. */ calibration?: number | null; /** Fixed-vs-perturbed accuracy gap, in [0,1]. */ contaminationDelta?: number | null; } const clamp01 = (x: number): number => Math.min(1, Math.max(0, x)); const clampIndex = (x: number): number => Math.min(THETA_SCALE.max, Math.max(THETA_SCALE.min, x)); /** Logistic squash of θ → [0,1]; P(correct) on a median (b=0, a=1) item. */ export function abilityToUnit(theta: number): number { return 1 / (1 + Math.exp(-theta)); } /** Rescale θ (± CI from SE) directly to the 0–1000 display scale. */ export function thetaToIndex(theta: number, thetaSe: number): ScoreWithCI { const score = clampIndex(THETA_SCALE.center + THETA_SCALE.slope * theta); const half = THETA_SCALE.slope * CI_Z * thetaSe; return { score: Math.round(score), scoreLow: Math.round(clampIndex(score - half)), scoreHigh: Math.round(clampIndex(score + half)), }; } export function contaminationResistance(delta: number): number { return clamp01(1 - delta / CONTAMINATION_DELTA_FLOOR); } /** * Blend available sub-metrics into a domain composite in [0,1], renormalizing * weights over the metrics actually present (a missing metric neither rewards * nor punishes). accuracy_irt is always required. */ export function domainComposite(m: DomainSubMetrics): number { const values: Partial> = { accuracy_irt: abilityToUnit(m.theta), }; if (m.consistency != null) values.consistency = clamp01(m.consistency); if (m.calibration != null) values.calibration = clamp01(m.calibration); if (m.contaminationDelta != null) values.contamination_resistance = contaminationResistance(m.contaminationDelta); let weightSum = 0; let acc = 0; for (const [key, value] of Object.entries(values) as [SubMetricKey, number][]) { const w = SUBMETRIC_WEIGHTS[key]; weightSum += w; acc += w * value; } return acc / weightSum; } /** * Domain composite with CI, propagated from θ SE via the delta method through * the accuracy term (the only stochastic fit parameter in the composite). */ export function domainScore(m: DomainSubMetrics): ScoreWithCI { const composite = domainComposite(m); const p = abilityToUnit(m.theta); const present: SubMetricKey[] = ['accuracy_irt']; if (m.consistency != null) present.push('consistency'); if (m.calibration != null) present.push('calibration'); if (m.contaminationDelta != null) present.push('contamination_resistance'); const weightSum = present.reduce((s, k) => s + SUBMETRIC_WEIGHTS[k], 0); const wAcc = SUBMETRIC_WEIGHTS.accuracy_irt / weightSum; const half = 1000 * wAcc * p * (1 - p) * CI_Z * m.thetaSe; const score = 1000 * composite; return { score: Math.round(score), scoreLow: Math.round(clampIndex(score - half)), scoreHigh: Math.round(clampIndex(score + half)), }; } /** * Global Index: weighted average of per-domain composites over the domains a * model was actually evaluated on (weights renormalized), rescaled to 0–1000. * CI combines domain CI half-widths in quadrature (domains fit independently). */ export function globalIndex(perDomain: Partial>): ScoreWithCI | null { const entries = Object.entries(perDomain) as [Domain, ScoreWithCI][]; if (entries.length === 0) return null; const weightSum = entries.reduce((s, [d]) => s + DOMAIN_WEIGHTS[d], 0); let score = 0; let varAcc = 0; for (const [d, s] of entries) { const w = DOMAIN_WEIGHTS[d] / weightSum; score += w * s.score; const half = (s.scoreHigh - s.scoreLow) / 2; varAcc += (w * half) ** 2; } const half = Math.sqrt(varAcc); return { score: Math.round(score), scoreLow: Math.round(clampIndex(score - half)), scoreHigh: Math.round(clampIndex(score + half)), }; } export interface ParetoPoint { slug: string; /** Quality score (higher better), e.g. Global Index. */ score: number; /** Cost in USD per 1k items (lower better). */ costPer1kItems: number; } /** Efficiency frontier: models not dominated on (score↑, cost↓). Never a blended number. */ export function paretoFrontier(points: ParetoPoint[]): ParetoPoint[] { const sorted = [...points].sort((a, b) => a.costPer1kItems - b.costPer1kItems || b.score - a.score); const frontier: ParetoPoint[] = []; let best = -Infinity; for (const p of sorted) { if (p.score > best) { frontier.push(p); best = p.score; } } return frontier; } /** Bradley-Terry strength → θ-like scale used by duel domains (log-strength standardized). */ export function btStrengthToTheta(logStrengths: number[]): number[] { const mean = logStrengths.reduce((a, b) => a + b, 0) / logStrengths.length; const sd = Math.sqrt(logStrengths.reduce((a, b) => a + (b - mean) ** 2, 0) / logStrengths.length) || 1; return logStrengths.map((s) => (s - mean) / sd); }