/** * llmindex.io — server-side data access for pages + public API * Author: Simon-Pierre Boucher * Contact: contact@spboucher.ai * License: Proprietary — © Simon-Pierre Boucher, all rights reserved * * INTEGRITY: this module NEVER selects EvalItem.answerKey / rubric. Item keys * and grading rubrics must not reach the client bundle or the public API. */ import 'server-only'; import { prisma, type ScoreRun } from '@llmindex/db'; import { GLOBAL_DOMAIN, INDEX_VERSION, paretoFrontier, type ParetoPoint } from '@llmindex/scoring'; export interface LeaderboardEntry { rank: number; slug: string; name: string; provider: string; score: number; scoreLow: number; scoreHigh: number; subMetrics: Record | null; } export interface LeaderboardData { run: Pick; entries: LeaderboardEntry[]; } /** * Latest displayable run — CURRENT index version only. Older-version runs * stay in the database as immutable audit history but never mix into the * displayed rankings. */ export async function getLatestRun(): Promise { try { const fit = await prisma.scoreRun.findFirst({ where: { kind: 'index_fit', indexVersion: INDEX_VERSION, status: { in: ['complete', 'degraded'] }, }, orderBy: { createdAt: 'desc' }, }); if (fit) return fit; return await prisma.scoreRun.findFirst({ where: { kind: 'demo_seed', indexVersion: INDEX_VERSION, status: 'complete' }, orderBy: { createdAt: 'desc' }, }); } catch { // DB unreachable (e.g. build-time prerender) — pages render an empty state. return null; } } export async function getLeaderboard( domain: string = GLOBAL_DOMAIN, limit = 100, offset = 0, ): Promise { const run = await getLatestRun(); if (!run) return null; const scores = await prisma.score.findMany({ where: { runId: run.id, domain }, orderBy: { score: 'desc' }, include: { model: { select: { slug: true, name: true, provider: true } } }, take: limit, skip: offset, }); return { run: { id: run.id, indexVersion: run.indexVersion, kind: run.kind, status: run.status, createdAt: run.createdAt, notes: run.notes, }, entries: scores.map((s, i) => ({ rank: offset + i + 1, slug: s.model.slug, name: s.model.name, provider: s.model.provider, score: s.score, scoreLow: s.scoreLow, scoreHigh: s.scoreHigh, subMetrics: (s.subMetrics as Record | null) ?? null, })), }; } export interface ModelProfile { slug: string; name: string; provider: string; contextLength: number | null; promptPricePerM: number | null; completionPricePerM: number | null; run: LeaderboardData['run']; global: LeaderboardEntry | null; domains: Array<{ domain: string; score: number; scoreLow: number; scoreHigh: number; subMetrics: Record | null; }>; runHistory: Array<{ runId: string; indexVersion: string; kind: string; createdAt: Date; score: number | null }>; } export async function getModelProfile(slug: string): Promise { const run = await getLatestRun(); if (!run) return null; const model = await prisma.model.findUnique({ where: { slug } }); if (!model) return null; const scores = await prisma.score.findMany({ where: { runId: run.id, modelId: model.id }, orderBy: { domain: 'asc' }, }); if (scores.length === 0) return null; const globalScore = scores.find((s) => s.domain === GLOBAL_DOMAIN) ?? null; const history = await prisma.score.findMany({ where: { modelId: model.id, domain: GLOBAL_DOMAIN }, include: { run: { select: { id: true, indexVersion: true, kind: true, createdAt: true } } }, orderBy: { run: { createdAt: 'desc' } }, take: 20, }); return { slug: model.slug, name: model.name, provider: model.provider, contextLength: model.contextLength, promptPricePerM: model.promptPricePerM, completionPricePerM: model.completionPricePerM, run: { id: run.id, indexVersion: run.indexVersion, kind: run.kind, status: run.status, createdAt: run.createdAt, notes: run.notes, }, global: globalScore ? { rank: 0, slug: model.slug, name: model.name, provider: model.provider, score: globalScore.score, scoreLow: globalScore.scoreLow, scoreHigh: globalScore.scoreHigh, subMetrics: null, } : null, domains: scores .filter((s) => s.domain !== GLOBAL_DOMAIN) .map((s) => ({ domain: s.domain, score: s.score, scoreLow: s.scoreLow, scoreHigh: s.scoreHigh, subMetrics: (s.subMetrics as Record | null) ?? null, })), runHistory: history.map((h) => ({ runId: h.run.id, indexVersion: h.run.indexVersion, kind: h.run.kind, createdAt: h.run.createdAt, score: h.score, })), }; } export interface EfficiencyPoint extends ParetoPoint { name: string; provider: string; scoreLow: number; scoreHigh: number; latencyP50: number | null; onFrontier: boolean; } export interface ResponseRow { domain: string; templateId: string; isAnchor: boolean; /** Anchor prompts stay hidden (longitudinal subset protection). */ prompt: string | null; answer: string | null; correct: boolean | null; confidence: number | null; latencyMs: number | null; costUsd: number | null; tokensOut: number | null; error: string | null; } /** * Full transparency: every graded answer of the model in the current index * version (latest response per item). answer keys are NEVER selected here. */ export async function getModelResponses(slug: string): Promise> { try { const rows = await prisma.modelResponse.findMany({ where: { model: { slug }, sampleIndex: 0, run: { kind: 'eval_batch', indexVersion: INDEX_VERSION, status: { in: ['complete', 'degraded'] } }, }, include: { item: { select: { domain: true, templateId: true, isAnchor: true, prompt: true } }, }, orderBy: { createdAt: 'desc' }, take: 800, }); const latestPerItem = new Map(); for (const r of rows) if (!latestPerItem.has(r.itemId)) latestPerItem.set(r.itemId, r); const grouped: Record = {}; for (const r of latestPerItem.values()) { const domain = r.item.domain; (grouped[domain] ??= []).push({ domain, templateId: r.item.templateId, isAnchor: r.item.isAnchor, prompt: r.item.isAnchor ? null : r.item.prompt, answer: r.answerExtracted, correct: r.correct, confidence: r.confidence, latencyMs: r.latencyMs, costUsd: r.costUsd, tokensOut: r.tokensOut, error: r.error, }); } return grouped; } catch { return {}; } } export interface DuelRow { domain: string; prompt: string; opponent: string; judge: string; outcome: 'win' | 'loss' | 'tie'; positionSwapped: boolean; myResponseExcerpt: string | null; createdAt: Date; } /** Judge-rated duels involving this model (current index version). */ export async function getModelDuels(slug: string): Promise { try { const duels = await prisma.pairwiseDuel.findMany({ where: { run: { indexVersion: INDEX_VERSION }, OR: [{ modelA: { slug } }, { modelB: { slug } }], }, include: { item: { select: { prompt: true } }, modelA: { select: { slug: true, name: true } }, modelB: { select: { slug: true, name: true } }, }, orderBy: { createdAt: 'desc' }, take: 60, }); return duels.map((d) => { const iAmA = d.modelA.slug === slug; const raw = d.rawJudgment as { responseA?: string; responseB?: string } | null; const outcome: DuelRow['outcome'] = d.winner === 'tie' ? 'tie' : (d.winner === 'a') === iAmA ? 'win' : 'loss'; return { domain: d.domain, prompt: d.item.prompt, opponent: iAmA ? d.modelB.name : d.modelA.name, judge: d.judgeSlug, outcome, positionSwapped: d.positionSwapped, myResponseExcerpt: (iAmA ? raw?.responseA : raw?.responseB)?.slice(0, 1200) ?? null, createdAt: d.createdAt, }; }); } catch { return []; } } /** Score vs cost-per-1k-items — rendered as a Pareto frontier, never blended. */ export async function getEfficiencyData(): Promise { const run = await getLatestRun(); if (!run) return []; const scores = await prisma.score.findMany({ where: { runId: run.id }, include: { model: { select: { slug: true, name: true, provider: true } } }, }); interface Acc { name: string; provider: string; global: { score: number; low: number; high: number } | null; costs: number[]; latencies: number[]; } const byModel = new Map(); for (const s of scores) { const entry: Acc = byModel.get(s.model.slug) ?? { name: s.model.name, provider: s.model.provider, global: null, costs: [], latencies: [], }; if (s.domain === GLOBAL_DOMAIN) entry.global = { score: s.score, low: s.scoreLow, high: s.scoreHigh }; const sub = s.subMetrics as Record | null; if (typeof sub?.cost_per_1k_items === 'number') entry.costs.push(sub.cost_per_1k_items); if (typeof sub?.latency_p50 === 'number') entry.latencies.push(sub.latency_p50); byModel.set(s.model.slug, entry); } const points: ParetoPoint[] = []; const meta = new Map(); for (const [slug, e] of byModel) { if (e.global === null || e.costs.length === 0) continue; meta.set(slug, e); points.push({ slug, score: e.global.score, costPer1kItems: e.costs.reduce((a, b) => a + b, 0) / e.costs.length, }); } const frontier = new Set(paretoFrontier(points).map((p) => p.slug)); const median = (arr: number[]): number | null => { if (!arr.length) return null; const s = [...arr].sort((a, b) => a - b); return s[Math.floor(s.length / 2)]!; }; return points .map((p) => { const e = meta.get(p.slug)!; return { ...p, name: e.name, provider: e.provider, scoreLow: e.global!.low, scoreHigh: e.global!.high, latencyP50: median(e.latencies), onFrontier: frontier.has(p.slug), }; }) .sort((a, b) => a.costPer1kItems - b.costPer1kItems); }