spb/llmindex Public

1/**2 * llmindex.io — server-side data access for pages + public API3 * Author:  Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * License: Proprietary — © Simon-Pierre Boucher, all rights reserved6 *7 * INTEGRITY: this module NEVER selects EvalItem.answerKey / rubric. Item keys8 * and grading rubrics must not reach the client bundle or the public API.9 */10import 'server-only';11import { prisma, type ScoreRun } from '@llmindex/db';12import { GLOBAL_DOMAIN, INDEX_VERSION, paretoFrontier, type ParetoPoint } from '@llmindex/scoring';1314export interface LeaderboardEntry {15  rank: number;16  slug: string;17  name: string;18  provider: string;19  score: number;20  scoreLow: number;21  scoreHigh: number;22  subMetrics: Record<string, number> | null;23}2425export interface LeaderboardData {26  run: Pick<ScoreRun, 'id' | 'indexVersion' | 'kind' | 'status' | 'createdAt' | 'notes'>;27  entries: LeaderboardEntry[];28}2930/**31 * Latest displayable run — CURRENT index version only. Older-version runs32 * stay in the database as immutable audit history but never mix into the33 * displayed rankings.34 */35export async function getLatestRun(): Promise<ScoreRun | null> {36  try {37    const fit = await prisma.scoreRun.findFirst({38      where: {39        kind: 'index_fit',40        indexVersion: INDEX_VERSION,41        status: { in: ['complete', 'degraded'] },42      },43      orderBy: { createdAt: 'desc' },44    });45    if (fit) return fit;46    return await prisma.scoreRun.findFirst({47      where: { kind: 'demo_seed', indexVersion: INDEX_VERSION, status: 'complete' },48      orderBy: { createdAt: 'desc' },49    });50  } catch {51    // DB unreachable (e.g. build-time prerender) — pages render an empty state.52    return null;53  }54}5556export async function getLeaderboard(57  domain: string = GLOBAL_DOMAIN,58  limit = 100,59  offset = 0,60): Promise<LeaderboardData | null> {61  const run = await getLatestRun();62  if (!run) return null;63  const scores = await prisma.score.findMany({64    where: { runId: run.id, domain },65    orderBy: { score: 'desc' },66    include: { model: { select: { slug: true, name: true, provider: true } } },67    take: limit,68    skip: offset,69  });70  return {71    run: {72      id: run.id,73      indexVersion: run.indexVersion,74      kind: run.kind,75      status: run.status,76      createdAt: run.createdAt,77      notes: run.notes,78    },79    entries: scores.map((s, i) => ({80      rank: offset + i + 1,81      slug: s.model.slug,82      name: s.model.name,83      provider: s.model.provider,84      score: s.score,85      scoreLow: s.scoreLow,86      scoreHigh: s.scoreHigh,87      subMetrics: (s.subMetrics as Record<string, number> | null) ?? null,88    })),89  };90}9192export interface ModelProfile {93  slug: string;94  name: string;95  provider: string;96  contextLength: number | null;97  promptPricePerM: number | null;98  completionPricePerM: number | null;99  run: LeaderboardData['run'];100  global: LeaderboardEntry | null;101  domains: Array<{102    domain: string;103    score: number;104    scoreLow: number;105    scoreHigh: number;106    subMetrics: Record<string, number> | null;107  }>;108  runHistory: Array<{ runId: string; indexVersion: string; kind: string; createdAt: Date; score: number | null }>;109}110111export async function getModelProfile(slug: string): Promise<ModelProfile | null> {112  const run = await getLatestRun();113  if (!run) return null;114  const model = await prisma.model.findUnique({ where: { slug } });115  if (!model) return null;116  const scores = await prisma.score.findMany({117    where: { runId: run.id, modelId: model.id },118    orderBy: { domain: 'asc' },119  });120  if (scores.length === 0) return null;121  const globalScore = scores.find((s) => s.domain === GLOBAL_DOMAIN) ?? null;122  const history = await prisma.score.findMany({123    where: { modelId: model.id, domain: GLOBAL_DOMAIN },124    include: { run: { select: { id: true, indexVersion: true, kind: true, createdAt: true } } },125    orderBy: { run: { createdAt: 'desc' } },126    take: 20,127  });128  return {129    slug: model.slug,130    name: model.name,131    provider: model.provider,132    contextLength: model.contextLength,133    promptPricePerM: model.promptPricePerM,134    completionPricePerM: model.completionPricePerM,135    run: {136      id: run.id,137      indexVersion: run.indexVersion,138      kind: run.kind,139      status: run.status,140      createdAt: run.createdAt,141      notes: run.notes,142    },143    global: globalScore144      ? {145          rank: 0,146          slug: model.slug,147          name: model.name,148          provider: model.provider,149          score: globalScore.score,150          scoreLow: globalScore.scoreLow,151          scoreHigh: globalScore.scoreHigh,152          subMetrics: null,153        }154      : null,155    domains: scores156      .filter((s) => s.domain !== GLOBAL_DOMAIN)157      .map((s) => ({158        domain: s.domain,159        score: s.score,160        scoreLow: s.scoreLow,161        scoreHigh: s.scoreHigh,162        subMetrics: (s.subMetrics as Record<string, number> | null) ?? null,163      })),164    runHistory: history.map((h) => ({165      runId: h.run.id,166      indexVersion: h.run.indexVersion,167      kind: h.run.kind,168      createdAt: h.run.createdAt,169      score: h.score,170    })),171  };172}173174export interface EfficiencyPoint extends ParetoPoint {175  name: string;176  provider: string;177  scoreLow: number;178  scoreHigh: number;179  latencyP50: number | null;180  onFrontier: boolean;181}182183export interface ResponseRow {184  domain: string;185  templateId: string;186  isAnchor: boolean;187  /** Anchor prompts stay hidden (longitudinal subset protection). */188  prompt: string | null;189  answer: string | null;190  correct: boolean | null;191  confidence: number | null;192  latencyMs: number | null;193  costUsd: number | null;194  tokensOut: number | null;195  error: string | null;196}197198/**199 * Full transparency: every graded answer of the model in the current index200 * version (latest response per item). answer keys are NEVER selected here.201 */202export async function getModelResponses(slug: string): Promise<Record<string, ResponseRow[]>> {203  try {204    const rows = await prisma.modelResponse.findMany({205      where: {206        model: { slug },207        sampleIndex: 0,208        run: { kind: 'eval_batch', indexVersion: INDEX_VERSION, status: { in: ['complete', 'degraded'] } },209      },210      include: {211        item: { select: { domain: true, templateId: true, isAnchor: true, prompt: true } },212      },213      orderBy: { createdAt: 'desc' },214      take: 800,215    });216    const latestPerItem = new Map<string, (typeof rows)[number]>();217    for (const r of rows) if (!latestPerItem.has(r.itemId)) latestPerItem.set(r.itemId, r);218    const grouped: Record<string, ResponseRow[]> = {};219    for (const r of latestPerItem.values()) {220      const domain = r.item.domain;221      (grouped[domain] ??= []).push({222        domain,223        templateId: r.item.templateId,224        isAnchor: r.item.isAnchor,225        prompt: r.item.isAnchor ? null : r.item.prompt,226        answer: r.answerExtracted,227        correct: r.correct,228        confidence: r.confidence,229        latencyMs: r.latencyMs,230        costUsd: r.costUsd,231        tokensOut: r.tokensOut,232        error: r.error,233      });234    }235    return grouped;236  } catch {237    return {};238  }239}240241export interface DuelRow {242  domain: string;243  prompt: string;244  opponent: string;245  judge: string;246  outcome: 'win' | 'loss' | 'tie';247  positionSwapped: boolean;248  myResponseExcerpt: string | null;249  createdAt: Date;250}251252/** Judge-rated duels involving this model (current index version). */253export async function getModelDuels(slug: string): Promise<DuelRow[]> {254  try {255    const duels = await prisma.pairwiseDuel.findMany({256      where: {257        run: { indexVersion: INDEX_VERSION },258        OR: [{ modelA: { slug } }, { modelB: { slug } }],259      },260      include: {261        item: { select: { prompt: true } },262        modelA: { select: { slug: true, name: true } },263        modelB: { select: { slug: true, name: true } },264      },265      orderBy: { createdAt: 'desc' },266      take: 60,267    });268    return duels.map((d) => {269      const iAmA = d.modelA.slug === slug;270      const raw = d.rawJudgment as { responseA?: string; responseB?: string } | null;271      const outcome: DuelRow['outcome'] =272        d.winner === 'tie' ? 'tie' : (d.winner === 'a') === iAmA ? 'win' : 'loss';273      return {274        domain: d.domain,275        prompt: d.item.prompt,276        opponent: iAmA ? d.modelB.name : d.modelA.name,277        judge: d.judgeSlug,278        outcome,279        positionSwapped: d.positionSwapped,280        myResponseExcerpt: (iAmA ? raw?.responseA : raw?.responseB)?.slice(0, 1200) ?? null,281        createdAt: d.createdAt,282      };283    });284  } catch {285    return [];286  }287}288289/** Score vs cost-per-1k-items — rendered as a Pareto frontier, never blended. */290export async function getEfficiencyData(): Promise<EfficiencyPoint[]> {291  const run = await getLatestRun();292  if (!run) return [];293  const scores = await prisma.score.findMany({294    where: { runId: run.id },295    include: { model: { select: { slug: true, name: true, provider: true } } },296  });297  interface Acc {298    name: string;299    provider: string;300    global: { score: number; low: number; high: number } | null;301    costs: number[];302    latencies: number[];303  }304  const byModel = new Map<string, Acc>();305  for (const s of scores) {306    const entry: Acc = byModel.get(s.model.slug) ?? {307      name: s.model.name,308      provider: s.model.provider,309      global: null,310      costs: [],311      latencies: [],312    };313    if (s.domain === GLOBAL_DOMAIN)314      entry.global = { score: s.score, low: s.scoreLow, high: s.scoreHigh };315    const sub = s.subMetrics as Record<string, number> | null;316    if (typeof sub?.cost_per_1k_items === 'number') entry.costs.push(sub.cost_per_1k_items);317    if (typeof sub?.latency_p50 === 'number') entry.latencies.push(sub.latency_p50);318    byModel.set(s.model.slug, entry);319  }320  const points: ParetoPoint[] = [];321  const meta = new Map<string, Acc>();322  for (const [slug, e] of byModel) {323    if (e.global === null || e.costs.length === 0) continue;324    meta.set(slug, e);325    points.push({326      slug,327      score: e.global.score,328      costPer1kItems: e.costs.reduce((a, b) => a + b, 0) / e.costs.length,329    });330  }331  const frontier = new Set(paretoFrontier(points).map((p) => p.slug));332  const median = (arr: number[]): number | null => {333    if (!arr.length) return null;334    const s = [...arr].sort((a, b) => a - b);335    return s[Math.floor(s.length / 2)]!;336  };337  return points338    .map((p) => {339      const e = meta.get(p.slug)!;340      return {341        ...p,342        name: e.name,343        provider: e.provider,344        scoreLow: e.global!.low,345        scoreHigh: e.global!.high,346        latencyP50: median(e.latencies),347        onFrontier: frontier.has(p.slug),348      };349    })350    .sort((a, b) => a.costPer1kItems - b.costPer1kItems);351}352