spb/llmindex Public
The discriminative, contamination-resistant, fully transparent LLM ranking — updated live.
TypeScript 77.9%
TeX 15.2%
Python 3.7%
SQL 1.4%
JavaScript 1.1%
Shell 0.5%
1/**2 * llmindex.io — live leaderboard: polls scores + benchmark progress, updates in place3 * Author: Simon-Pierre Boucher4 * Contact: contact@spboucher.ai5 * License: Proprietary — © Simon-Pierre Boucher, all rights reserved6 *7 * Results appear gradually, one model at a time, as the online benchmark8 * finishes each model and the IRT refit publishes a new score run.9 */10'use client';1112import { useEffect, useRef, useState } from 'react';13import Link from 'next/link';14import { DemoBanner, ScoreBar, ScoreValue, formatMs, formatUsd } from '@llmindex/ui';15import { ProviderLogo } from './ProviderLogo';1617export interface ApiEntry {18 rank: number;19 model: string;20 name: string;21 provider: string;22 score: number;23 score_low: number;24 score_high: number;25 sub_metrics?: Record<string, number | null> | null;26}2728export interface ApiRun {29 id: string;30 kind: string;31 status: string;32 created_at: string;33 notes?: string | null;34}3536interface Progress {37 active: boolean;38 models?: string[];39 completed?: string[];40 failed?: string[];41 current?: {42 model: string;43 domain: string;44 domain_index: number;45 domains_total: number;46 calls_done: number;47 calls_total: number;48 } | null;49 currents?: Array<NonNullable<Progress['current']>>;50 note?: string;51}5253export interface LiveLeaderboardProps {54 domain?: string;55 initialRun: ApiRun | null;56 initialEntries: ApiEntry[];57 indexVersion: string;58}5960const POLL_MS = 5000;6162function ProgressBanner({ progress }: { progress: Progress }) {63 const total = progress.models?.length ?? 0;64 const nDone = progress.completed?.length ?? 0;65 const nFailed = progress.failed?.length ?? 0;66 const done = nDone + nFailed;67 const overallPct = total > 0 ? Math.round((100 * done) / total) : 0;68 const lanes =69 progress.currents && progress.currents.length > 070 ? progress.currents71 : progress.current72 ? [progress.current]73 : [];74 const runningModels = new Set(lanes.map((l) => l.model)).size;75 const recent = (progress.completed ?? []).slice(-5).reverse();76 const remaining = Math.max(0, total - done - runningModels);7778 return (79 <div className="overflow-hidden rounded-2xl border border-emerald-200 bg-gradient-to-br from-emerald-50 via-white to-cyan-50">80 {/* header */}81 <div className="flex flex-wrap items-center gap-x-4 gap-y-2 px-4 py-3 sm:px-5">82 <span className="flex items-center gap-2 text-sm font-semibold text-emerald-700">83 <span className="relative flex h-2.5 w-2.5">84 <span className="absolute inline-flex h-full w-full animate-ping rounded-full bg-emerald-400 opacity-60" />85 <span className="relative inline-flex h-2.5 w-2.5 rounded-full bg-emerald-400" />86 </span>87 Live benchmark88 </span>89 <span className="text-xs text-zinc-600">90 <span className="font-semibold text-zinc-900 tabular-nums">{done}</span>91 <span className="text-zinc-500">/{total} models</span>92 {nFailed > 0 && <span className="ml-2 text-red-400">{nFailed} failed</span>}93 </span>94 <span className="ml-auto text-2xl font-bold tabular-nums text-zinc-900">{overallPct}%</span>95 </div>96 {/* overall bar */}97 <div className="h-1 w-full bg-zinc-100">98 <div99 className="h-1 bg-gradient-to-r from-emerald-500 to-cyan-500 transition-all duration-700"100 style={{ width: `${overallPct}%` }}101 />102 </div>103104 {/* active evaluation lanes (parallel) */}105 {lanes.length > 0 && (106 <div className="divide-y divide-zinc-100">107 {lanes.map((lane, i) => {108 const pct = lane.calls_total > 0 ? Math.round((100 * lane.calls_done) / lane.calls_total) : 0;109 return (110 <div key={`${lane.model}-${lane.domain}-${i}`} className="flex items-center gap-3 px-4 py-2.5 sm:px-5">111 <div className="flex h-9 w-9 shrink-0 items-center justify-center rounded-lg border border-zinc-200 bg-white">112 <ProviderLogo provider={lane.model.split('/')[0] ?? ''} size={20} />113 </div>114 <div className="min-w-0 flex-1">115 <div className="flex flex-wrap items-baseline gap-x-2.5 gap-y-0.5">116 <span className="truncate text-[13px] font-semibold text-zinc-900">117 {lane.model.split('/')[1] ?? lane.model}118 </span>119 <span className="text-[11px] capitalize text-cyan-700">120 {lane.domain.replaceAll('_', ' ')}121 </span>122 <span className="ml-auto text-[10px] tabular-nums text-zinc-500">123 {lane.calls_done}/{lane.calls_total}124 </span>125 </div>126 <div className="mt-1 h-1 w-full overflow-hidden rounded-full bg-zinc-200">127 <div128 className="h-1 rounded-full bg-gradient-to-r from-emerald-500 to-cyan-500 transition-all duration-700"129 style={{ width: `${pct}%` }}130 />131 </div>132 </div>133 </div>134 );135 })}136 </div>137 )}138139 {/* recent completions + queue count */}140 {(recent.length > 0 || remaining > 0) && (141 <div className="flex flex-wrap items-center gap-1.5 border-t border-zinc-200 px-4 py-2.5 text-[11px] sm:px-5">142 {recent.length > 0 && <span className="mr-1 text-zinc-500">just finished:</span>}143 {recent.map((m) => (144 <span145 key={m}146 title={m}147 className="inline-flex max-w-[10rem] items-center gap-1.5 rounded-full border border-emerald-200 bg-emerald-50 px-2 py-0.5 text-emerald-700"148 >149 <ProviderLogo provider={m.split('/')[0] ?? ''} size={11} />150 <span className="truncate">{m.split('/')[1] ?? m}</span>151 </span>152 ))}153 {remaining > 0 && (154 <span className="ml-auto rounded-full bg-zinc-100 px-2.5 py-0.5 text-zinc-600">155 {remaining} in queue156 </span>157 )}158 </div>159 )}160 </div>161 );162}163164export function LiveLeaderboard({ domain, initialRun, initialEntries, indexVersion }: LiveLeaderboardProps) {165 const [entries, setEntries] = useState<ApiEntry[]>(initialEntries);166 const [run, setRun] = useState<ApiRun | null>(initialRun);167 const [progress, setProgress] = useState<Progress>({ active: false });168 const [fresh, setFresh] = useState<Set<string>>(new Set());169 const knownRef = useRef<Set<string>>(new Set(initialEntries.map((e) => e.model)));170171 useEffect(() => {172 let cancelled = false;173 async function tick() {174 try {175 const base = domain ? `/api/v1/leaderboard/${domain}` : '/api/v1/leaderboard';176 const [pRes, lRes] = await Promise.all([177 fetch('/api/v1/benchmark/progress', { cache: 'no-store' }),178 fetch(`${base}?limit=100`, { cache: 'no-store' }),179 ]);180 if (cancelled) return;181 if (pRes.ok) {182 const p = (await pRes.json()) as { progress: Progress };183 setProgress(p.progress ?? { active: false });184 }185 if (lRes.ok) {186 const body = (await lRes.json()) as { run: ApiRun; entries: ApiEntry[] };187 if (body.entries) {188 const newcomers = body.entries189 .map((e) => e.model)190 .filter((slug) => !knownRef.current.has(slug));191 if (newcomers.length > 0) {192 setFresh(new Set(newcomers));193 for (const slug of newcomers) knownRef.current.add(slug);194 setTimeout(() => setFresh(new Set()), 4000);195 }196 setEntries(body.entries);197 setRun(body.run);198 }199 }200 } catch {201 /* transient poll failure — keep last state */202 }203 }204 tick();205 const id = setInterval(tick, POLL_MS);206 return () => {207 cancelled = true;208 clearInterval(id);209 };210 }, [domain]);211212 const showSub = Boolean(domain);213214 return (215 <div className="space-y-4">216 {progress.active && <ProgressBanner progress={progress} />}217 {run?.kind === 'demo_seed' && <DemoBanner />}218219 <div className="flex flex-wrap items-baseline justify-between gap-2">220 <h2 className="text-lg font-semibold text-zinc-900 sm:text-xl">221 {domain ? `${domain.replaceAll('_', ' ')} ranking` : 'Global Index'}222 </h2>223 {run && (224 <span className="text-xs text-zinc-500">225 index v{indexVersion} · run {run.id.slice(0, 8)} ·{' '}226 {new Date(run.created_at).toISOString().slice(0, 16).replace('T', ' ')} UTC227 </span>228 )}229 </div>230231 {entries.length === 0 ? (232 <p className="rounded-xl border border-zinc-200 p-6 text-sm text-zinc-600">233 No scores yet{progress.active ? ' — the live benchmark is warming up; first results appear once two models complete.' : '.'}234 </p>235 ) : (236 <>237 {/* Desktop table */}238 <div className="hidden overflow-x-auto md:block">239 <table className="w-full text-sm">240 <thead>241 <tr className="border-b border-zinc-200 text-left text-xs uppercase tracking-wide text-zinc-500">242 <th className="py-2 pr-3">#</th>243 <th className="py-2 pr-3">Model</th>244 <th className="py-2 pr-3">Provider</th>245 <th className="py-2 pr-3 text-right">Score (95% CI)</th>246 <th className="w-1/4 py-2 pr-3"></th>247 {showSub && (248 <>249 <th className="py-2 pr-3 text-right">Consist.</th>250 <th className="py-2 pr-3 text-right">Calib.</th>251 <th className="py-2 pr-3 text-right">Contam. Δ</th>252 <th className="py-2 pr-3 text-right">p50</th>253 <th className="py-2 pr-3 text-right">$/1k</th>254 </>255 )}256 </tr>257 </thead>258 <tbody>259 {entries.map((e) => (260 <tr261 key={e.model}262 className={263 'border-b border-zinc-100 transition-colors hover:bg-zinc-50 ' +264 (fresh.has(e.model) ? 'animate-row-in bg-emerald-50' : '')265 }266 >267 <td className="py-2.5 pr-3 text-zinc-500">{e.rank}</td>268 <td className="py-2.5 pr-3">269 <Link270 href={`/models/${e.model}`}271 className="flex items-center gap-2.5 font-medium text-zinc-900 hover:text-emerald-600"272 >273 <ProviderLogo provider={e.provider} size={18} />274 {e.name}275 </Link>276 </td>277 <td className="py-2.5 pr-3 text-zinc-600">{e.provider}</td>278 <td className="py-2.5 pr-3 text-right">279 <ScoreValue score={e.score} scoreLow={e.score_low} scoreHigh={e.score_high} />280 </td>281 <td className="py-2.5 pr-3">282 <ScoreBar score={e.score} scoreLow={e.score_low} scoreHigh={e.score_high} />283 </td>284 {showSub && (285 <>286 <td className="py-2.5 pr-3 text-right tabular-nums text-zinc-600">287 {e.sub_metrics?.consistency?.toFixed(2) ?? '—'}288 </td>289 <td className="py-2.5 pr-3 text-right tabular-nums text-zinc-600">290 {e.sub_metrics?.calibration?.toFixed(2) ?? '—'}291 </td>292 <td className="py-2.5 pr-3 text-right tabular-nums text-zinc-600">293 {e.sub_metrics?.contamination_delta?.toFixed(3) ?? '—'}294 </td>295 <td className="py-2.5 pr-3 text-right tabular-nums text-zinc-600">296 {formatMs(e.sub_metrics?.latency_p50)}297 </td>298 <td className="py-2.5 pr-3 text-right tabular-nums text-zinc-600">299 {formatUsd(e.sub_metrics?.cost_per_1k_items)}300 </td>301 </>302 )}303 </tr>304 ))}305 </tbody>306 </table>307 </div>308309 {/* Mobile cards */}310 <ul className="space-y-2.5 md:hidden">311 {entries.map((e) => (312 <li313 key={e.model}314 className={315 'rounded-xl border border-zinc-200 bg-white p-3.5 ' +316 (fresh.has(e.model) ? 'animate-row-in border-emerald-300' : '')317 }318 >319 <div className="flex items-center justify-between gap-3">320 <div className="flex min-w-0 items-center gap-2.5">321 <ProviderLogo provider={e.provider} size={22} />322 <div className="min-w-0">323 <Link href={`/models/${e.model}`} className="block truncate font-medium text-zinc-900">324 <span className="mr-2 text-zinc-500">#{e.rank}</span>325 {e.name}326 </Link>327 <span className="text-xs text-zinc-500">{e.provider}</span>328 </div>329 </div>330 <div className="shrink-0 text-right">331 <ScoreValue score={e.score} scoreLow={e.score_low} scoreHigh={e.score_high} />332 </div>333 </div>334 <div className="mt-2.5">335 <ScoreBar score={e.score} scoreLow={e.score_low} scoreHigh={e.score_high} />336 </div>337 {showSub && e.sub_metrics && (338 <div className="mt-2.5 flex flex-wrap gap-1.5 text-[10px] text-zinc-600">339 {e.sub_metrics.consistency != null && (340 <span className="rounded-full bg-zinc-200 px-2 py-0.5">341 consist {e.sub_metrics.consistency.toFixed(2)}342 </span>343 )}344 {e.sub_metrics.calibration != null && (345 <span className="rounded-full bg-zinc-200 px-2 py-0.5">346 calib {e.sub_metrics.calibration.toFixed(2)}347 </span>348 )}349 {e.sub_metrics.contamination_delta != null && (350 <span className="rounded-full bg-zinc-200 px-2 py-0.5">351 Δ {e.sub_metrics.contamination_delta.toFixed(3)}352 </span>353 )}354 {e.sub_metrics.latency_p50 != null && (355 <span className="rounded-full bg-zinc-200 px-2 py-0.5">356 {formatMs(e.sub_metrics.latency_p50)}357 </span>358 )}359 {e.sub_metrics.cost_per_1k_items != null && (360 <span className="rounded-full bg-zinc-200 px-2 py-0.5">361 {formatUsd(e.sub_metrics.cost_per_1k_items)}/1k362 </span>363 )}364 </div>365 )}366 </li>367 ))}368 </ul>369 </>370 )}371 </div>372 );373}374