spb/llmindex Public
The discriminative, contamination-resistant, fully transparent LLM ranking — updated live.
TypeScript 77.9%
TeX 15.2%
Python 3.7%
SQL 1.4%
JavaScript 1.1%
Shell 0.5%
1/**2 * llmindex.io — index refit: response matrices → Python 2PL/BT fit → score run3 * Author: Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * License: Proprietary — © Simon-Pierre Boucher, all rights reserved6 */7import { spawnSync } from 'node:child_process';8import { createHash } from 'node:crypto';9import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';10import { dirname, join } from 'node:path';11import { fileURLToPath } from 'node:url';12import { prisma } from '@llmindex/db';13import {14 DUEL_DOMAINS,15 GLOBAL_DOMAIN,16 INDEX_VERSION,17 IRT_DOMAINS,18 IRT_HYPERPARAMS,19 domainScore,20 globalIndex,21 type Domain,22 type ScoreWithCI,23} from '@llmindex/scoring';2425const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..', '..');26const RUNS_DIR = join(REPO_ROOT, 'data', 'runs');27const FIT_SCRIPT = join(REPO_ROOT, 'apps', 'psychometrics', 'fit.py');28// Prefer the psychometrics venv (numpy) when present; fall back to system python3.29const VENV_PYTHON = join(REPO_ROOT, 'apps', 'psychometrics', '.venv', 'bin', 'python3');30const PYTHON = existsSync(VENV_PYTHON) ? VENV_PYTHON : 'python3';3132const MIN_MODELS = 2;33const MIN_ITEMS = 10;3435interface DomainMatrix {36 models: string[]; // slugs37 items: string[]; // eval_item ids38 is_anchor: boolean[];39 /** responses[m][i] ∈ 0 | 1 | null */40 responses: (0 | 1 | null)[][];41}4243interface FitOutput {44 domains: Record<45 string,46 {47 models: Array<{ slug: string; theta: number; se: number }>;48 items: Array<{ id: string; a: number; b: number }>;49 diagnostics: Record<string, unknown>;50 }51 >;52 duel_domains?: Record<53 string,54 {55 models: Array<{ slug: string; theta: number; se: number }>;56 diagnostics: Record<string, unknown>;57 }58 >;59}6061interface DuelMatrix {62 models: string[];63 /** wins[i][j] = wins of i over j (ties pre-encoded as 0.5 each). */64 wins: number[][];65}6667interface AuxMetrics {68 consistency: number | null;69 calibration: number | null;70 contaminationDelta: number | null;71 latencyP50: number | null;72 costPer1kItems: number | null;73}7475function percentile(values: number[], p: number): number | null {76 if (values.length === 0) return null;77 const sorted = [...values].sort((a, b) => a - b);78 return sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))]!;79}8081/** Expected calibration error over 10 confidence bins → calibration = 1 − ECE. */82function calibrationFrom(pairs: Array<{ confidence: number; correct: boolean }>): number | null {83 if (pairs.length < 10) return null;84 const bins = Array.from({ length: 10 }, () => ({ n: 0, conf: 0, acc: 0 }));85 for (const p of pairs) {86 const b = bins[Math.min(9, Math.floor(p.confidence * 10))]!;87 b.n += 1;88 b.conf += p.confidence;89 b.acc += p.correct ? 1 : 0;90 }91 let ece = 0;92 for (const b of bins) {93 if (b.n === 0) continue;94 ece += (b.n / pairs.length) * Math.abs(b.acc / b.n - b.conf / b.n);95 }96 return 1 - ece;97}9899export async function runRefit(): Promise<{ runId: string } | null> {100 // Degraded batches are excluded from ranking until re-run (§6); only101 // batches of the CURRENT index version enter a fit (methodology coherence).102 const batches = await prisma.scoreRun.findMany({103 where: { kind: 'eval_batch', status: 'complete', indexVersion: INDEX_VERSION },104 select: { id: true },105 });106 const duelBatches = await prisma.scoreRun.findMany({107 where: { kind: 'duel_batch', status: 'complete', indexVersion: INDEX_VERSION },108 select: { id: true },109 });110 if (batches.length === 0 && duelBatches.length === 0) {111 console.warn('[refit] no complete batches for this index version — nothing to fit');112 return null;113 }114 const batchIds = batches.map((b) => b.id);115116 const matrices: Record<string, DomainMatrix> = {};117 const aux: Record<string, Record<string, AuxMetrics>> = {}; // domain → slug → metrics118119 for (const domain of IRT_DOMAINS) {120 const responses = await prisma.modelResponse.findMany({121 where: {122 runId: { in: batchIds },123 sampleIndex: 0,124 correct: { not: null },125 item: { domain },126 },127 include: {128 model: { select: { slug: true } },129 item: { select: { id: true, isAnchor: true } },130 },131 orderBy: { createdAt: 'asc' },132 });133 if (responses.length === 0) continue;134135 // Latest response per model×item wins (re-runs supersede).136 const latest = new Map<string, (typeof responses)[number]>();137 for (const r of responses) latest.set(`${r.model.slug}::${r.item.id}`, r);138139 const modelSlugs = [...new Set([...latest.values()].map((r) => r.model.slug))].sort();140 const itemIds = [...new Set([...latest.values()].map((r) => r.item.id))].sort();141 if (modelSlugs.length < MIN_MODELS || itemIds.length < MIN_ITEMS) {142 console.warn(143 `[refit] domain ${domain}: ${modelSlugs.length} models × ${itemIds.length} items — below minimum, skipped`,144 );145 continue;146 }147 const anchorSet = new Map(148 [...latest.values()].map((r) => [r.item.id, r.item.isAnchor] as const),149 );150 matrices[domain] = {151 models: modelSlugs,152 items: itemIds,153 is_anchor: itemIds.map((id) => anchorSet.get(id) ?? false),154 responses: modelSlugs.map((slug) =>155 itemIds.map((itemId) => {156 const r = latest.get(`${slug}::${itemId}`);157 return r ? ((r.correct ? 1 : 0) as 0 | 1) : null;158 }),159 ),160 };161162 // Auxiliary sub-metrics per model for this domain.163 aux[domain] = {};164 for (const slug of modelSlugs) {165 const mine = [...latest.values()].filter((r) => r.model.slug === slug);166 const anchorGraded = mine.filter((r) => r.item.isAnchor);167 const freshGraded = mine.filter((r) => !r.item.isAnchor);168 const acc = (rs: typeof mine): number | null =>169 rs.length ? rs.filter((r) => r.correct).length / rs.length : null;170 const anchorAcc = acc(anchorGraded);171 const freshAcc = acc(freshGraded);172173 // Consistency: same item, k samples — fraction agreeing with the modal answer.174 const kSamples = await prisma.modelResponse.findMany({175 where: {176 runId: { in: batchIds },177 model: { slug },178 item: { domain },179 answerExtracted: { not: null },180 },181 select: { itemId: true, answerExtracted: true },182 });183 const byItem = new Map<string, string[]>();184 for (const s of kSamples) {185 byItem.set(s.itemId, [...(byItem.get(s.itemId) ?? []), s.answerExtracted!]);186 }187 const consistencies: number[] = [];188 for (const answers of byItem.values()) {189 if (answers.length < 2) continue;190 const counts = new Map<string, number>();191 for (const a of answers) counts.set(a, (counts.get(a) ?? 0) + 1);192 consistencies.push(Math.max(...counts.values()) / answers.length);193 }194195 const confPairs = mine196 .filter((r) => r.confidence != null)197 .map((r) => ({ confidence: r.confidence!, correct: r.correct === true }));198 const latencies = mine.map((r) => r.latencyMs).filter((v): v is number => v != null);199 const costs = mine.map((r) => r.costUsd).filter((v): v is number => v != null);200201 aux[domain][slug] = {202 consistency: consistencies.length203 ? consistencies.reduce((a, b) => a + b, 0) / consistencies.length204 : null,205 calibration: calibrationFrom(confPairs),206 contaminationDelta:207 anchorAcc != null && freshAcc != null ? Math.max(0, anchorAcc - freshAcc) : null,208 latencyP50: percentile(latencies, 0.5),209 costPer1kItems: costs.length210 ? (costs.reduce((a, b) => a + b, 0) / costs.length) * 1000211 : null,212 };213 }214 }215216 // Bradley-Terry matrices for judged duel domains (writing, safety, svg_design).217 const duelMatrices: Record<string, DuelMatrix> = {};218 if (duelBatches.length > 0) {219 for (const domain of DUEL_DOMAINS) {220 const duels = await prisma.pairwiseDuel.findMany({221 where: { runId: { in: duelBatches.map((b) => b.id) }, domain },222 include: {223 modelA: { select: { slug: true } },224 modelB: { select: { slug: true } },225 },226 });227 if (duels.length < 10) continue;228 const slugs = [...new Set(duels.flatMap((d) => [d.modelA.slug, d.modelB.slug]))].sort();229 if (slugs.length < MIN_MODELS) continue;230 const idx = new Map(slugs.map((s, i) => [s, i]));231 const wins = slugs.map(() => slugs.map(() => 0));232 for (const d of duels) {233 const a = idx.get(d.modelA.slug)!;234 const b = idx.get(d.modelB.slug)!;235 if (d.winner === 'a') wins[a]![b]! += 1;236 else if (d.winner === 'b') wins[b]![a]! += 1;237 else {238 wins[a]![b]! += 0.5;239 wins[b]![a]! += 0.5;240 }241 }242 duelMatrices[domain] = { models: slugs, wins };243 }244 }245246 const domainsFitted = Object.keys(matrices);247 const duelDomainsFitted = Object.keys(duelMatrices);248 if (domainsFitted.length === 0 && duelDomainsFitted.length === 0) {249 console.warn('[refit] no domain met the minimum matrix size — aborting');250 return null;251 }252253 mkdirSync(RUNS_DIR, { recursive: true });254 const stamp = new Date().toISOString().replace(/[:.]/g, '-');255 const inputPath = join(RUNS_DIR, `refit-${stamp}-input.json`);256 const outputPath = join(RUNS_DIR, `refit-${stamp}-output.json`);257 writeFileSync(258 inputPath,259 JSON.stringify(260 { hyperparams: IRT_HYPERPARAMS, domains: matrices, duel_domains: duelMatrices },261 null,262 2,263 ),264 );265266 console.log(`[refit] fitting ${domainsFitted.length} domain(s) via ${FIT_SCRIPT}`);267 const py = spawnSync(PYTHON, [FIT_SCRIPT, '--input', inputPath, '--output', outputPath], {268 stdio: 'inherit',269 cwd: REPO_ROOT,270 });271 if (py.status !== 0) throw new Error(`psychometrics fit failed (exit ${py.status})`);272273 const fit = JSON.parse(readFileSync(outputPath, 'utf8')) as FitOutput;274275 const itemSetHash = createHash('sha256')276 .update(277 domainsFitted.map((d) => matrices[d]!.items.join(',')).join('|') +278 '||' +279 duelDomainsFitted.map((d) => `${d}:${duelMatrices[d]!.models.join(',')}`).join('|'),280 )281 .digest('hex');282 const allSlugs = [283 ...new Set([284 ...domainsFitted.flatMap((d) => matrices[d]!.models),285 ...duelDomainsFitted.flatMap((d) => duelMatrices[d]!.models),286 ]),287 ].sort();288289 const run = await prisma.scoreRun.create({290 data: {291 indexVersion: INDEX_VERSION,292 kind: 'index_fit',293 status: 'running',294 itemSetHash,295 modelSet: allSlugs,296 fitDiagnostics: Object.fromEntries([297 ...domainsFitted.map((d) => [d, fit.domains[d]?.diagnostics ?? {}]),298 ...duelDomainsFitted.map((d) => [`duel:${d}`, fit.duel_domains?.[d]?.diagnostics ?? {}]),299 ]) as object,300 notes: `sources=${batchIds.length} eval batches + ${duelBatches.length} duel batches`,301 },302 });303304 const perModelDomain = new Map<string, Partial<Record<Domain, ScoreWithCI>>>();305306 for (const domain of domainsFitted) {307 const fitted = fit.domains[domain];308 if (!fitted) continue;309 const model = await Promise.all(310 fitted.models.map(async (m) => {311 const dbModel = await prisma.model.findUnique({ where: { slug: m.slug } });312 if (!dbModel) return null;313 const metrics = aux[domain]?.[m.slug];314 const s = domainScore({315 theta: m.theta,316 thetaSe: m.se,317 consistency: metrics?.consistency,318 calibration: metrics?.calibration,319 contaminationDelta: metrics?.contaminationDelta,320 });321 await prisma.score.create({322 data: {323 runId: run.id,324 modelId: dbModel.id,325 domain,326 score: s.score,327 scoreLow: s.scoreLow,328 scoreHigh: s.scoreHigh,329 subMetrics: {330 accuracy_irt: Number((1 / (1 + Math.exp(-m.theta))).toFixed(4)),331 theta: Number(m.theta.toFixed(4)),332 theta_se: Number(m.se.toFixed(4)),333 consistency: metrics?.consistency ?? null,334 calibration: metrics?.calibration ?? null,335 contamination_delta: metrics?.contaminationDelta ?? null,336 latency_p50: metrics?.latencyP50 ?? null,337 cost_per_1k_items: metrics?.costPer1kItems ?? null,338 },339 },340 });341 const acc = perModelDomain.get(m.slug) ?? {};342 acc[domain as Domain] = s;343 perModelDomain.set(m.slug, acc);344 return m.slug;345 }),346 );347 void model;348349 // Persist item parameters; discrimination hygiene (§7.4).350 for (const item of fitted.items) {351 const flag =352 item.a < IRT_HYPERPARAMS.minDiscrimination ||353 Math.abs(item.b) > IRT_HYPERPARAMS.maxAbsDifficultyLogits;354 await prisma.evalItem.update({355 where: { id: item.id },356 data: {357 irtA: item.a,358 irtB: item.b,359 ...(flag ? { status: 'flagged_for_retirement' } : {}),360 },361 });362 }363 }364365 // Duel domains: Bradley-Terry log-strengths arrive standardized θ-like.366 for (const domain of duelDomainsFitted) {367 const fitted = fit.duel_domains?.[domain];368 if (!fitted) continue;369 for (const m of fitted.models) {370 const dbModel = await prisma.model.findUnique({ where: { slug: m.slug } });371 if (!dbModel) continue;372 const s = domainScore({ theta: m.theta, thetaSe: m.se });373 await prisma.score.create({374 data: {375 runId: run.id,376 modelId: dbModel.id,377 domain,378 score: s.score,379 scoreLow: s.scoreLow,380 scoreHigh: s.scoreHigh,381 subMetrics: {382 accuracy_irt: Number((1 / (1 + Math.exp(-m.theta))).toFixed(4)),383 theta: Number(m.theta.toFixed(4)),384 theta_se: Number(m.se.toFixed(4)),385 method: 'bradley_terry',386 },387 },388 });389 const acc = perModelDomain.get(m.slug) ?? {};390 acc[domain as Domain] = s;391 perModelDomain.set(m.slug, acc);392 }393 }394395 for (const [slug, domains] of perModelDomain) {396 const g = globalIndex(domains);397 if (!g) continue;398 const dbModel = await prisma.model.findUnique({ where: { slug } });399 if (!dbModel) continue;400 await prisma.score.create({401 data: {402 runId: run.id,403 modelId: dbModel.id,404 domain: GLOBAL_DOMAIN,405 score: g.score,406 scoreLow: g.scoreLow,407 scoreHigh: g.scoreHigh,408 subMetrics: { domains_covered: Object.keys(domains) },409 },410 });411 }412413 await prisma.scoreRun.update({414 where: { id: run.id },415 data: { status: 'complete', completedAt: new Date() },416 });417 console.log(`[refit] index_fit run ${run.id} complete (${domainsFitted.join(', ')})`);418 return { runId: run.id };419}420