feat(web): deep-detail methodology, privacy policy, terms of service, live efficiency frontier
- Methodology: scoring mathematics (2PL MAP, Fisher SE, BT MM + SE, composite formulas), item lifecycle & bank hygiene, full judge protocol, known limitations, versioning/audit section, contact block - New /privacy and /terms pages (contact: Simon-Pierre Boucher, contact@spboucher.ai), footer links - Efficiency frontier + hero stats now poll live (/api/v1/efficiency) - Rankings display current INDEX_VERSION only (old runs remain as immutable audit history) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Showing 8 changed files with +529 and −77
added
apps/web/app/api/v1/efficiency/route.ts
+27 −0
@@ -0,0 +1,27 @@ | ||
| 1 | +/** | |
| 2 | + * llmindex.io — GET /api/v1/efficiency (Pareto frontier data, live) | |
| 3 | + * Author: Simon-Pierre Boucher | |
| 4 | + * Contact: contact@spboucher.ai | |
| 5 | + * License: Proprietary — © Simon-Pierre Boucher, all rights reserved | |
| 6 | + */ | |
| 7 | +import { type NextRequest } from 'next/server'; | |
| 8 | +import { getEfficiencyData } from '@/lib/data'; | |
| 9 | +import { apiJson } from '@/lib/api'; | |
| 10 | +import { rateLimit } from '@/lib/rate-limit'; | |
| 11 | + | |
| 12 | +export const dynamic = 'force-dynamic'; | |
| 13 | + | |
| 14 | +export async function GET(req: NextRequest) { | |
| 15 | + const limited = await rateLimit(req); | |
| 16 | + if (limited) return limited; | |
| 17 | + const points = await getEfficiencyData(); | |
| 18 | + return apiJson({ | |
| 19 | + points: points.map((p) => ({ | |
| 20 | + model: p.slug, | |
| 21 | + name: p.name, | |
| 22 | + score: p.score, | |
| 23 | + cost_per_1k_items: Number(p.costPer1kItems.toFixed(4)), | |
| 24 | + on_frontier: p.onFrontier, | |
| 25 | + })), | |
| 26 | + }); | |
| 27 | +} | |
modified
apps/web/app/layout.tsx
+21 −5
@@ -52,13 +52,29 @@ export default function RootLayout({ children }: { children: React.ReactNode }) | ||
| 52 | 52 | </header> |
| 53 | 53 | <main className="mx-auto max-w-6xl px-4 py-6 sm:py-10">{children}</main> |
| 54 | 54 | <footer className="border-t border-zinc-200 px-4 py-6 text-center text-xs leading-relaxed text-zinc-500"> |
| 55 | − © {new Date().getFullYear()} LLM Index — maintained by Simon-Pierre Boucher ·{' '} | |
| 56 | − <a className="underline hover:text-zinc-700" href="mailto:contact@spboucher.ai"> | |
| 57 | − contact@spboucher.ai | |
| 58 | − </a> | |
| 55 | + <span> | |
| 56 | + © {new Date().getFullYear()} LLM Index — maintained by Simon-Pierre Boucher ·{' '} | |
| 57 | + <a className="underline hover:text-zinc-700" href="mailto:contact@spboucher.ai"> | |
| 58 | + contact@spboucher.ai | |
| 59 | + </a> | |
| 60 | + </span> | |
| 59 | 61 | <br className="sm:hidden" /> |
| 60 | 62 | <span className="hidden sm:inline"> · </span> |
| 61 | − All scores carry 95% confidence intervals · Methodology is fully public | |
| 63 | + <span>All scores carry 95% confidence intervals</span> | |
| 64 | + <div className="mt-2 flex flex-wrap items-center justify-center gap-x-4 gap-y-1"> | |
| 65 | + <Link href="/methodology" className="underline hover:text-zinc-700"> | |
| 66 | + Methodology | |
| 67 | + </Link> | |
| 68 | + <Link href="/privacy" className="underline hover:text-zinc-700"> | |
| 69 | + Privacy Policy | |
| 70 | + </Link> | |
| 71 | + <Link href="/terms" className="underline hover:text-zinc-700"> | |
| 72 | + Terms of Service | |
| 73 | + </Link> | |
| 74 | + <a href="mailto:contact@spboucher.ai" className="underline hover:text-zinc-700"> | |
| 75 | + Contact | |
| 76 | + </a> | |
| 77 | + </div> | |
| 62 | 78 | </footer> |
| 63 | 79 | </body> |
| 64 | 80 | </html> |
modified
apps/web/app/methodology/page.tsx
+122 −0
@@ -131,6 +131,103 @@ export default function MethodologyPage() { | ||
| 131 | 131 | ))} |
| 132 | 132 | </section> |
| 133 | 133 | |
| 134 | + <section className="space-y-4"> | |
| 135 | + <h2 className="text-xl font-semibold text-zinc-900">Scoring mathematics</h2> | |
| 136 | + <div className="grid gap-4 md:grid-cols-2"> | |
| 137 | + <div className="rounded-xl border border-zinc-200 bg-white p-5"> | |
| 138 | + <h3 className="mb-2 text-sm font-semibold text-zinc-900">Item response model (2PL)</h3> | |
| 139 | + <pre className="overflow-x-auto rounded-lg bg-zinc-100 p-3 text-xs text-zinc-700">{`P(correct | θ, a, b) = 1 / (1 + exp(−a·(θ − b))) | |
| 140 | + | |
| 141 | +MAP objective (per domain, missing-aware): | |
| 142 | + L = Σ observed [ x·log P + (1−x)·log(1−P) ] | |
| 143 | + − θ²/2σθ² − b²/2σb² − (log a)²/2σloga² | |
| 144 | + priors: θ ~ N(0,1) · b ~ N(0,1.5) · log a ~ N(0,0.5) | |
| 145 | + | |
| 146 | +Optimizer: Adam, warm-started from accuracy logits, | |
| 147 | +θ recentered each step (scale pinned by priors), | |
| 148 | +max 500 iterations, tolerance 1e-6.`}</pre> | |
| 149 | + </div> | |
| 150 | + <div className="rounded-xl border border-zinc-200 bg-white p-5"> | |
| 151 | + <h3 className="mb-2 text-sm font-semibold text-zinc-900">Uncertainty & display scale</h3> | |
| 152 | + <pre className="overflow-x-auto rounded-lg bg-zinc-100 p-3 text-xs text-zinc-700">{`SE(θ) = 1 / √( Σᵢ aᵢ²·P·(1−P) + 1/σθ² ) | |
| 153 | + (Fisher information + prior precision) | |
| 154 | + | |
| 155 | +Domain composite (all terms in [0,1]): | |
| 156 | + C = w_acc·σ(θ) + w_con·consistency | |
| 157 | + + w_cal·calibration + w_res·resistance | |
| 158 | + weights renormalize over measured terms | |
| 159 | + | |
| 160 | +Domain score = 1000·C, CI via delta method | |
| 161 | + through the accuracy term | |
| 162 | +Global Index = 1000·Σ_d w_d·C_d (equal w_d, | |
| 163 | + renormalized over covered domains) | |
| 164 | +Global CI = per-domain half-widths combined | |
| 165 | + in quadrature (independent fits)`}</pre> | |
| 166 | + </div> | |
| 167 | + <div className="rounded-xl border border-zinc-200 bg-white p-5"> | |
| 168 | + <h3 className="mb-2 text-sm font-semibold text-zinc-900">Bradley-Terry (duel domains)</h3> | |
| 169 | + <pre className="overflow-x-auto rounded-lg bg-zinc-100 p-3 text-xs text-zinc-700">{`P(i beats j) = pᵢ / (pᵢ + pⱼ) | |
| 170 | +ties → half-win to each side | |
| 171 | +MM update: pᵢ ← Wᵢ / Σⱼ Nᵢⱼ/(pᵢ+pⱼ) | |
| 172 | +damping: +0.1 phantom win per pair (finite | |
| 173 | +strengths for isolated models) | |
| 174 | +SE from Fisher info: Iᵢᵢ = Σⱼ Nᵢⱼ·pᵢⱼ·(1−pᵢⱼ) | |
| 175 | +log-strengths standardized → same θ scale | |
| 176 | +as IRT domains before rescaling.`}</pre> | |
| 177 | + </div> | |
| 178 | + <div className="rounded-xl border border-zinc-200 bg-white p-5"> | |
| 179 | + <h3 className="mb-2 text-sm font-semibold text-zinc-900">Sub-metric definitions</h3> | |
| 180 | + <pre className="overflow-x-auto rounded-lg bg-zinc-100 p-3 text-xs text-zinc-700">{`consistency = mean over items of | |
| 181 | + (# samples agreeing with modal answer / k) | |
| 182 | + k samples at the model's default temperature | |
| 183 | + | |
| 184 | +calibration = 1 − ECE (10 equal-width bins over | |
| 185 | + the model's self-reported 0-100 confidence) | |
| 186 | + | |
| 187 | +contamination_delta = acc(anchors) − acc(fresh) | |
| 188 | +resistance = clamp01(1 − Δ/0.2) | |
| 189 | + (a 20-point memorization gap ⇒ zero credit) | |
| 190 | + | |
| 191 | +latency_p50, cost_per_1k_items: measured per | |
| 192 | +call, published, NEVER blended into quality.`}</pre> | |
| 193 | + </div> | |
| 194 | + </div> | |
| 195 | + </section> | |
| 196 | + | |
| 197 | + <section className="space-y-3"> | |
| 198 | + <h2 className="text-xl font-semibold text-zinc-900">Item lifecycle & bank hygiene</h2> | |
| 199 | + <ol className="list-decimal space-y-2 pl-5 text-sm leading-relaxed text-zinc-600"> | |
| 200 | + <li><span className="font-medium text-zinc-800">Generation</span> — every scored batch draws a fresh batch seed; each item derives from <code>seed:domain:index</code> through a deterministic PRNG (mulberry32 over an FNV-1a hash), so a batch is fully reproducible from its recorded seed yet unpredictable in advance.</li> | |
| 201 | + <li><span className="font-medium text-zinc-800">Anchors</span> — positions below the anchor fraction (≤20%, capped in code) derive from a <em>fixed</em> seed stream (<code>anchor:domain:index</code>) — byte-identical across every run, forever. They exist only to measure longitudinal drift and the memorization gap.</li> | |
| 202 | + <li><span className="font-medium text-zinc-800">Administration</span> — scored sample at temperature 0 with a 16,384-token completion budget; k additional samples at the model's default temperature for consistency. Full audit row per call: exact request params, raw response, tokens, measured latency, measured cost.</li> | |
| 203 | + <li><span className="font-medium text-zinc-800">Grading</span> — mechanical only: normalized exact match, numeric tolerance (1e-6 relative), canonical-JSON equality for call sequences, exact multi-line match for terminal output, constraint-checker stacks for instruction following. No LLM ever grades a scored item.</li> | |
| 204 | + <li><span className="font-medium text-zinc-800">Retirement</span> — after every refit, items with discrimination a < {IRT_HYPERPARAMS.minDiscrimination} or |b| > {IRT_HYPERPARAMS.maxAbsDifficultyLogits} logits are flagged <code>flagged_for_retirement</code>; flagged parameter regions are reviewed before the next run. Dead items never silently dilute the index.</li> | |
| 205 | + <li><span className="font-medium text-zinc-800">Degradation</span> — a batch with >2% failed calls is marked <code>degraded</code>, shown as such, and excluded from every fit until re-run. Truncated completions are unscored, never counted wrong.</li> | |
| 206 | + </ol> | |
| 207 | + </section> | |
| 208 | + | |
| 209 | + <section className="space-y-3"> | |
| 210 | + <h2 className="text-xl font-semibold text-zinc-900">Judge protocol (duel domains) — full detail</h2> | |
| 211 | + <ul className="list-disc space-y-2 pl-5 text-sm leading-relaxed text-zinc-600"> | |
| 212 | + <li><span className="font-medium text-zinc-800">Panel</span>: three judge models from at least two different providers, configured explicitly (never hardcoded); a model never judges a duel it participates in — ineligible judges are excluded per-duel, and a duel with fewer than two eligible judges is skipped entirely.</li> | |
| 213 | + <li><span className="font-medium text-zinc-800">Position swap</span>: judges alternate which response they see first; the swap is recorded on every verdict row and verdicts are un-swapped before aggregation, so position bias is both mitigated and measurable.</li> | |
| 214 | + <li><span className="font-medium text-zinc-800">Verdicts</span>: judges output a single-token verdict (1 / 2 / TIE) at temperature 0; unparseable verdicts count conservatively as ties; empty or degenerate model responses can never win.</li> | |
| 215 | + <li><span className="font-medium text-zinc-800">Anti-verbosity</span>: the judge prompt explicitly forbids rewarding length or style over substance; for SVG duels judges evaluate geometric fidelity, brand-color accuracy and vector cleanliness of the code.</li> | |
| 216 | + <li><span className="font-medium text-zinc-800">Published diagnostics</span>: per run — judged-verdict count, pair count, and full-panel agreement rate, stored in the run's immutable <code>fit_diagnostics</code>.</li> | |
| 217 | + </ul> | |
| 218 | + </section> | |
| 219 | + | |
| 220 | + <section className="space-y-3"> | |
| 221 | + <h2 className="text-xl font-semibold text-zinc-900">Known limitations (read this too)</h2> | |
| 222 | + <ul className="list-disc space-y-2 pl-5 text-sm leading-relaxed text-zinc-600"> | |
| 223 | + <li><span className="font-medium text-zinc-800">IRT with few examinees</span>: item-parameter recovery degrades below ~100 models. We deliberately rank a wide roster (135 models incl. small anchors) to stabilize the scale, and CIs are published precisely so you never over-read a 10-point gap.</li> | |
| 224 | + <li><span className="font-medium text-zinc-800">Judge subjectivity</span>: duel domains inherit judge preferences even with a cross-provider panel, swaps and agreement reporting. Duel scores are best read comparatively, not absolutely.</li> | |
| 225 | + <li><span className="font-medium text-zinc-800">Provider variance</span>: models are reached through OpenRouter routing; latency (and occasionally behavior) can vary by upstream provider. Latency is reported as measured p50 per run, not a hardware-normalized figure.</li> | |
| 226 | + <li><span className="font-medium text-zinc-800">Template scope</span>: generators cover targeted, mechanically-gradeable slices of each capability — deliberately narrow and deep rather than broad and judge-dependent. The template list is public; judge for yourself what is and isn't measured.</li> | |
| 227 | + <li><span className="font-medium text-zinc-800">Single-turn agentic</span>: v0.2 grades planned call sequences in one shot; multi-turn interactive episodes with injected tool errors are on the roadmap.</li> | |
| 228 | + </ul> | |
| 229 | + </section> | |
| 230 | + | |
| 134 | 231 | <section className="space-y-3"> |
| 135 | 232 | <h2 className="text-xl font-semibold text-zinc-900">The pipeline, end to end</h2> |
| 136 | 233 | <ol className="list-decimal space-y-2 pl-5 text-sm leading-relaxed text-zinc-600"> |
@@ -254,6 +351,17 @@ export default function MethodologyPage() { | ||
| 254 | 351 | </pre> |
| 255 | 352 | </section> |
| 256 | 353 | |
| 354 | + <section className="space-y-3"> | |
| 355 | + <h2 className="text-xl font-semibold text-zinc-900">Versioning, reproducibility & audit trail</h2> | |
| 356 | + <ul className="list-disc space-y-2 pl-5 text-sm leading-relaxed text-zinc-600"> | |
| 357 | + <li>Every methodology change bumps the semver <code>INDEX_VERSION</code> with an entry in the public changelog; scores from different versions are never mixed in one fit.</li> | |
| 358 | + <li>Every displayed number traces to an immutable <code>score_runs</code> row: item-set hash, model set, index version, fit diagnostics, timestamps. Historical runs are never edited.</li> | |
| 359 | + <li>Every model call stores its exact request parameters, raw response, token usage, measured latency and measured cost — a ranking without stored raw responses would be invalid by our own rules.</li> | |
| 360 | + <li>Weights, hyperparameters and domain definitions live in exactly one source file and are served machine-readable at <a className="underline hover:text-zinc-900" href="/api/v1/methodology">/api/v1/methodology</a>.</li> | |
| 361 | + <li>Batches are reproducible from their recorded seeds; anchors are byte-stable across all runs.</li> | |
| 362 | + </ul> | |
| 363 | + </section> | |
| 364 | + | |
| 257 | 365 | <section className="space-y-3"> |
| 258 | 366 | <h2 className="text-xl font-semibold text-zinc-900">References & influences</h2> |
| 259 | 367 | <p className="max-w-3xl text-sm text-zinc-600"> |
@@ -275,6 +383,20 @@ export default function MethodologyPage() { | ||
| 275 | 383 | ))} |
| 276 | 384 | </ul> |
| 277 | 385 | </section> |
| 386 | + | |
| 387 | + <section className="rounded-xl border border-zinc-200 bg-white p-5"> | |
| 388 | + <h2 className="mb-2 text-xl font-semibold text-zinc-900">Contact & corrections</h2> | |
| 389 | + <p className="text-sm leading-relaxed text-zinc-600"> | |
| 390 | + Methodology questions, disputes about a score, retirement appeals for an item family, or | |
| 391 | + requests to add a model:{' '} | |
| 392 | + <span className="font-medium text-zinc-800">Simon-Pierre Boucher</span> ·{' '} | |
| 393 | + <a className="underline hover:text-emerald-600" href="mailto:contact@spboucher.ai"> | |
| 394 | + contact@spboucher.ai | |
| 395 | + </a> | |
| 396 | + . Substantiated corrections are applied in a new score run — never by editing history — | |
| 397 | + and acknowledged in the changelog. | |
| 398 | + </p> | |
| 399 | + </section> | |
| 278 | 400 | </div> |
| 279 | 401 | ); |
| 280 | 402 | } |
modified
apps/web/app/page.tsx
+14 −69
@@ -6,9 +6,8 @@ | ||
| 6 | 6 | */ |
| 7 | 7 | import Link from 'next/link'; |
| 8 | 8 | import { DOMAINS, INDEX_VERSION } from '@llmindex/scoring'; |
| 9 | −import { formatUsd } from '@llmindex/ui'; | |
| 10 | 9 | import { LiveLeaderboard, type ApiEntry, type ApiRun } from '@/components/LiveLeaderboard'; |
| 11 | −import { ProviderLogo } from '@/components/ProviderLogo'; | |
| 10 | +import { EfficiencyLive, HomeStats, type EffPoint } from '@/components/EfficiencyLive'; | |
| 12 | 11 | import { getEfficiencyData, getLeaderboard } from '@/lib/data'; |
| 13 | 12 | |
| 14 | 13 | export const revalidate = 300; |
@@ -34,6 +33,13 @@ export default async function HomePage() { | ||
| 34 | 33 | score_low: e.scoreLow, |
| 35 | 34 | score_high: e.scoreHigh, |
| 36 | 35 | })); |
| 36 | + const initialEfficiency: EffPoint[] = efficiency.map((p) => ({ | |
| 37 | + model: p.slug, | |
| 38 | + name: p.name, | |
| 39 | + score: p.score, | |
| 40 | + cost_per_1k_items: p.costPer1kItems, | |
| 41 | + on_frontier: p.onFrontier, | |
| 42 | + })); | |
| 37 | 43 | |
| 38 | 44 | return ( |
| 39 | 45 | <div className="space-y-10 sm:space-y-14"> |
@@ -51,26 +57,11 @@ export default async function HomePage() { | ||
| 51 | 57 | benchmarks, no leaked test sets. Results stream in live as each model finishes its |
| 52 | 58 | evaluation. |
| 53 | 59 | </p> |
| 54 | − <div className="flex flex-wrap gap-x-6 gap-y-2 pt-2 text-sm"> | |
| 55 | − <span className="flex items-center gap-2 text-zinc-700"> | |
| 56 | − <span className="relative flex h-2 w-2"> | |
| 57 | − <span className="absolute inline-flex h-full w-full animate-ping rounded-full bg-emerald-400 opacity-60" /> | |
| 58 | − <span className="relative inline-flex h-2 w-2 rounded-full bg-emerald-400" /> | |
| 59 | − </span> | |
| 60 | − updates live | |
| 61 | − </span> | |
| 62 | − <span className="text-zinc-600"> | |
| 63 | − <span className="font-semibold text-zinc-900 tabular-nums">{initialEntries.length}</span>{' '} | |
| 64 | − models scored | |
| 65 | − </span> | |
| 66 | − <span className="text-zinc-600"> | |
| 67 | − <span className="font-semibold text-zinc-900 tabular-nums">{DOMAINS.length}</span>{' '} | |
| 68 | − domains | |
| 69 | − </span> | |
| 70 | − <span className="text-zinc-600"> | |
| 71 | − IRT 2PL · 95% CI · v{INDEX_VERSION} | |
| 72 | − </span> | |
| 73 | − </div> | |
| 60 | + <HomeStats | |
| 61 | + initialCount={initialEntries.length} | |
| 62 | + domains={DOMAINS.length} | |
| 63 | + version={INDEX_VERSION} | |
| 64 | + /> | |
| 74 | 65 | <div className="flex flex-wrap gap-2 pt-1"> |
| 75 | 66 | {DOMAINS.map((d) => ( |
| 76 | 67 | <Link |
@@ -92,53 +83,7 @@ export default async function HomePage() { | ||
| 92 | 83 | /> |
| 93 | 84 | </section> |
| 94 | 85 | |
| 95 | − {efficiency.length > 0 && ( | |
| 96 | − <section className="space-y-3"> | |
| 97 | − <h2 className="text-lg font-semibold text-zinc-900 sm:text-xl">Efficiency frontier</h2> | |
| 98 | − <p className="text-sm text-zinc-500"> | |
| 99 | − Score vs. cost per 1k items. Frontier models are not dominated on both axes — never | |
| 100 | − collapsed into a single blended number. | |
| 101 | − </p> | |
| 102 | − <div className="overflow-x-auto rounded-xl border border-zinc-200"> | |
| 103 | − <table className="w-full min-w-[28rem] text-sm"> | |
| 104 | − <thead> | |
| 105 | − <tr className="border-b border-zinc-200 text-left text-xs uppercase tracking-wide text-zinc-500"> | |
| 106 | − <th className="px-3 py-2">Model</th> | |
| 107 | − <th className="px-3 py-2 text-right">Global Index</th> | |
| 108 | − <th className="px-3 py-2 text-right">Cost / 1k items</th> | |
| 109 | − <th className="px-3 py-2">Pareto</th> | |
| 110 | − </tr> | |
| 111 | − </thead> | |
| 112 | − <tbody> | |
| 113 | − {efficiency.map((p) => ( | |
| 114 | − <tr key={p.slug} className="border-b border-zinc-100 last:border-0"> | |
| 115 | − <td className="px-3 py-2"> | |
| 116 | − <Link | |
| 117 | − href={`/models/${p.slug}`} | |
| 118 | − className="flex items-center gap-2 hover:text-emerald-600" | |
| 119 | − > | |
| 120 | − <ProviderLogo provider={p.slug.split('/')[0] ?? ''} size={16} /> | |
| 121 | − {p.name} | |
| 122 | − </Link> | |
| 123 | − </td> | |
| 124 | − <td className="px-3 py-2 text-right tabular-nums">{p.score}</td> | |
| 125 | − <td className="px-3 py-2 text-right tabular-nums">{formatUsd(p.costPer1kItems)}</td> | |
| 126 | − <td className="px-3 py-2"> | |
| 127 | − {p.onFrontier ? ( | |
| 128 | − <span className="rounded-full bg-emerald-100 px-2 py-0.5 text-xs text-emerald-700"> | |
| 129 | − frontier | |
| 130 | − </span> | |
| 131 | − ) : ( | |
| 132 | − <span className="text-xs text-zinc-400">dominated</span> | |
| 133 | − )} | |
| 134 | − </td> | |
| 135 | − </tr> | |
| 136 | − ))} | |
| 137 | − </tbody> | |
| 138 | − </table> | |
| 139 | − </div> | |
| 140 | − </section> | |
| 141 | − )} | |
| 86 | + <EfficiencyLive initial={initialEfficiency} /> | |
| 142 | 87 | </div> |
| 143 | 88 | ); |
| 144 | 89 | } |
added
apps/web/app/privacy/page.tsx
+82 −0
@@ -0,0 +1,82 @@ | ||
| 1 | +/** | |
| 2 | + * llmindex.io — privacy policy | |
| 3 | + * Author: Simon-Pierre Boucher | |
| 4 | + * Contact: contact@spboucher.ai | |
| 5 | + * License: Proprietary — © Simon-Pierre Boucher, all rights reserved | |
| 6 | + */ | |
| 7 | +import type { Metadata } from 'next'; | |
| 8 | + | |
| 9 | +export const metadata: Metadata = { title: 'Privacy Policy' }; | |
| 10 | +export const revalidate = 86400; | |
| 11 | + | |
| 12 | +const UPDATED = '2026-08-05'; | |
| 13 | + | |
| 14 | +export default function PrivacyPage() { | |
| 15 | + return ( | |
| 16 | + <div className="mx-auto max-w-3xl space-y-8"> | |
| 17 | + <div className="space-y-2"> | |
| 18 | + <h1 className="text-2xl font-bold text-zinc-900 sm:text-3xl">Privacy Policy</h1> | |
| 19 | + <p className="text-sm text-zinc-500">Last updated: {UPDATED}</p> | |
| 20 | + </div> | |
| 21 | + | |
| 22 | + <section className="space-y-3 text-sm leading-relaxed text-zinc-600"> | |
| 23 | + <h2 className="text-lg font-semibold text-zinc-900">Who we are</h2> | |
| 24 | + <p> | |
| 25 | + LLM Index (www.llmindex.io) is operated by <span className="font-medium text-zinc-800">Simon-Pierre Boucher</span>{' '} | |
| 26 | + (“we”, “us”). Contact:{' '} | |
| 27 | + <a className="underline hover:text-emerald-600" href="mailto:contact@spboucher.ai">contact@spboucher.ai</a>. | |
| 28 | + </p> | |
| 29 | + </section> | |
| 30 | + | |
| 31 | + <section className="space-y-3 text-sm leading-relaxed text-zinc-600"> | |
| 32 | + <h2 className="text-lg font-semibold text-zinc-900">What we collect — and what we don't</h2> | |
| 33 | + <ul className="list-disc space-y-2 pl-5"> | |
| 34 | + <li><span className="font-medium text-zinc-800">No accounts, no profiles.</span> The site requires no registration and stores no user profiles.</li> | |
| 35 | + <li><span className="font-medium text-zinc-800">No advertising trackers, no third-party analytics.</span> We do not run ad networks, marketing pixels, or fingerprinting scripts, and we set no advertising cookies.</li> | |
| 36 | + <li><span className="font-medium text-zinc-800">Server logs & rate limiting.</span> Like virtually every website, our infrastructure processes IP addresses and standard request metadata (user agent, requested URL, timestamp) to serve pages, prevent abuse, and enforce the public API rate limits (60 requests/minute anonymously). Rate-limit counters are keyed on a transient identifier and expire automatically within minutes; routine logs are rotated and not used to identify individuals.</li> | |
| 37 | + <li><span className="font-medium text-zinc-800">Network providers.</span> Traffic transits our hosting and tunneling providers (e.g., ngrok), which maintain their own operational logs under their own policies.</li> | |
| 38 | + <li><span className="font-medium text-zinc-800">API keys.</span> If we issue you an API key for the higher rate tier, we store the key and its usage counters — nothing more.</li> | |
| 39 | + <li><span className="font-medium text-zinc-800">Email.</span> If you write to us, we keep the correspondence for as long as needed to handle your request.</li> | |
| 40 | + </ul> | |
| 41 | + </section> | |
| 42 | + | |
| 43 | + <section className="space-y-3 text-sm leading-relaxed text-zinc-600"> | |
| 44 | + <h2 className="text-lg font-semibold text-zinc-900">What the benchmark data contains</h2> | |
| 45 | + <p> | |
| 46 | + The rankings, scores, model responses and judge verdicts published on this site are | |
| 47 | + generated by us, about <em>AI models</em> — they contain no personal data about visitors. | |
| 48 | + Evaluation prompts are synthetically generated; no visitor input is ever sent to any AI | |
| 49 | + model. | |
| 50 | + </p> | |
| 51 | + </section> | |
| 52 | + | |
| 53 | + <section className="space-y-3 text-sm leading-relaxed text-zinc-600"> | |
| 54 | + <h2 className="text-lg font-semibold text-zinc-900">Legal bases & sharing</h2> | |
| 55 | + <p> | |
| 56 | + We process the minimal technical data described above on the basis of legitimate interest | |
| 57 | + (operating and protecting a public website). We do not sell or rent any data. We disclose | |
| 58 | + data only if legally required. | |
| 59 | + </p> | |
| 60 | + </section> | |
| 61 | + | |
| 62 | + <section className="space-y-3 text-sm leading-relaxed text-zinc-600"> | |
| 63 | + <h2 className="text-lg font-semibold text-zinc-900">Your rights</h2> | |
| 64 | + <p> | |
| 65 | + Depending on your jurisdiction (e.g., GDPR, PIPEDA, Québec Law 25), you may have rights of | |
| 66 | + access, rectification, deletion and objection regarding personal data. Given how little we | |
| 67 | + collect, most requests resolve trivially — write to{' '} | |
| 68 | + <a className="underline hover:text-emerald-600" href="mailto:contact@spboucher.ai">contact@spboucher.ai</a>{' '} | |
| 69 | + and we will respond within 30 days. | |
| 70 | + </p> | |
| 71 | + </section> | |
| 72 | + | |
| 73 | + <section className="space-y-3 text-sm leading-relaxed text-zinc-600"> | |
| 74 | + <h2 className="text-lg font-semibold text-zinc-900">Changes</h2> | |
| 75 | + <p> | |
| 76 | + We may update this policy; material changes will be reflected by the date above. Continued | |
| 77 | + use of the site after changes constitutes acceptance. | |
| 78 | + </p> | |
| 79 | + </section> | |
| 80 | + </div> | |
| 81 | + ); | |
| 82 | +} | |
added
apps/web/app/terms/page.tsx
+109 −0
@@ -0,0 +1,109 @@ | ||
| 1 | +/** | |
| 2 | + * llmindex.io — terms of service | |
| 3 | + * Author: Simon-Pierre Boucher | |
| 4 | + * Contact: contact@spboucher.ai | |
| 5 | + * License: Proprietary — © Simon-Pierre Boucher, all rights reserved | |
| 6 | + */ | |
| 7 | +import type { Metadata } from 'next'; | |
| 8 | + | |
| 9 | +export const metadata: Metadata = { title: 'Terms of Service' }; | |
| 10 | +export const revalidate = 86400; | |
| 11 | + | |
| 12 | +const UPDATED = '2026-08-05'; | |
| 13 | + | |
| 14 | +export default function TermsPage() { | |
| 15 | + return ( | |
| 16 | + <div className="mx-auto max-w-3xl space-y-8"> | |
| 17 | + <div className="space-y-2"> | |
| 18 | + <h1 className="text-2xl font-bold text-zinc-900 sm:text-3xl">Terms of Service</h1> | |
| 19 | + <p className="text-sm text-zinc-500">Last updated: {UPDATED}</p> | |
| 20 | + </div> | |
| 21 | + | |
| 22 | + <section className="space-y-3 text-sm leading-relaxed text-zinc-600"> | |
| 23 | + <h2 className="text-lg font-semibold text-zinc-900">1. Who provides this service</h2> | |
| 24 | + <p> | |
| 25 | + LLM Index (www.llmindex.io, including its public API — the “Service”) is | |
| 26 | + operated by <span className="font-medium text-zinc-800">Simon-Pierre Boucher</span>{' '} | |
| 27 | + (“we”, “us”). Contact:{' '} | |
| 28 | + <a className="underline hover:text-emerald-600" href="mailto:contact@spboucher.ai">contact@spboucher.ai</a>. | |
| 29 | + By using the Service you accept these terms. | |
| 30 | + </p> | |
| 31 | + </section> | |
| 32 | + | |
| 33 | + <section className="space-y-3 text-sm leading-relaxed text-zinc-600"> | |
| 34 | + <h2 className="text-lg font-semibold text-zinc-900">2. What the Service is</h2> | |
| 35 | + <p> | |
| 36 | + LLM Index publishes independent, methodology-driven evaluations of large language models: | |
| 37 | + scores, confidence intervals, sub-metrics, model responses, judge verdicts and an | |
| 38 | + efficiency frontier. Scores are statistical estimates produced by the published | |
| 39 | + methodology — they are <em>opinions grounded in a documented measurement process</em>, | |
| 40 | + not statements of objective fact about any model or vendor, and they change as new runs | |
| 41 | + complete and the methodology evolves (semver-versioned with a public changelog). | |
| 42 | + </p> | |
| 43 | + </section> | |
| 44 | + | |
| 45 | + <section className="space-y-3 text-sm leading-relaxed text-zinc-600"> | |
| 46 | + <h2 className="text-lg font-semibold text-zinc-900">3. Acceptable use</h2> | |
| 47 | + <ul className="list-disc space-y-2 pl-5"> | |
| 48 | + <li>Respect the published API rate limits (60 requests/minute anonymous; 600 with an issued key). Do not circumvent limits, scrape aggressively, disrupt, probe or overload the Service.</li> | |
| 49 | + <li>Do not attempt to extract non-public material (answer keys, anchor item instantiations, credentials) or to interfere with live benchmark runs.</li> | |
| 50 | + <li>Do not misrepresent LLM Index data as your own measurement, and do not present altered figures as ours.</li> | |
| 51 | + </ul> | |
| 52 | + </section> | |
| 53 | + | |
| 54 | + <section className="space-y-3 text-sm leading-relaxed text-zinc-600"> | |
| 55 | + <h2 className="text-lg font-semibold text-zinc-900">4. Using our data & attribution</h2> | |
| 56 | + <p> | |
| 57 | + You may quote, chart and cite scores and rankings from the Service — including | |
| 58 | + commercially — provided you attribute them to <span className="font-medium text-zinc-800"> | |
| 59 | + “LLM Index (llmindex.io)”</span> with the index version shown alongside the | |
| 60 | + data. Bulk redistribution of the dataset, or products substantially reproducing the | |
| 61 | + Service, require prior written permission. The platform's source code, design, logo | |
| 62 | + and content are © Simon-Pierre Boucher, all rights reserved. | |
| 63 | + </p> | |
| 64 | + </section> | |
| 65 | + | |
| 66 | + <section className="space-y-3 text-sm leading-relaxed text-zinc-600"> | |
| 67 | + <h2 className="text-lg font-semibold text-zinc-900">5. Third-party names</h2> | |
| 68 | + <p> | |
| 69 | + Model and provider names and logos appear solely to identify the evaluated systems | |
| 70 | + (nominative use). They remain trademarks of their respective owners; no affiliation, | |
| 71 | + sponsorship or endorsement is implied — by them of us, or by us of them. | |
| 72 | + </p> | |
| 73 | + </section> | |
| 74 | + | |
| 75 | + <section className="space-y-3 text-sm leading-relaxed text-zinc-600"> | |
| 76 | + <h2 className="text-lg font-semibold text-zinc-900">6. No warranty · limitation of liability</h2> | |
| 77 | + <p> | |
| 78 | + The Service is provided <em>“as is”</em> and <em>“as available”</em>, | |
| 79 | + without warranties of any kind, express or implied — including accuracy, completeness, | |
| 80 | + fitness for a particular purpose or uninterrupted availability. Do not base high-stakes | |
| 81 | + decisions solely on an index score. To the maximum extent permitted by law, our total | |
| 82 | + liability arising from the use of the Service is limited to the amount you paid us for it | |
| 83 | + (zero for the free Service). Nothing here excludes liability that cannot legally be | |
| 84 | + excluded. | |
| 85 | + </p> | |
| 86 | + </section> | |
| 87 | + | |
| 88 | + <section className="space-y-3 text-sm leading-relaxed text-zinc-600"> | |
| 89 | + <h2 className="text-lg font-semibold text-zinc-900">7. Corrections & disputes about scores</h2> | |
| 90 | + <p> | |
| 91 | + Model vendors and users may dispute a result by writing to{' '} | |
| 92 | + <a className="underline hover:text-emerald-600" href="mailto:contact@spboucher.ai">contact@spboucher.ai</a>{' '} | |
| 93 | + with specifics. Substantiated issues are corrected through a <em>new</em> score run | |
| 94 | + (history is immutable) and acknowledged in the public changelog. | |
| 95 | + </p> | |
| 96 | + </section> | |
| 97 | + | |
| 98 | + <section className="space-y-3 text-sm leading-relaxed text-zinc-600"> | |
| 99 | + <h2 className="text-lg font-semibold text-zinc-900">8. Changes, suspension, governing law</h2> | |
| 100 | + <p> | |
| 101 | + We may modify or discontinue the Service or these terms at any time; the date above | |
| 102 | + reflects the current version. We may suspend access that violates these terms. These | |
| 103 | + terms are governed by the laws of the Province of Québec and the federal laws of Canada | |
| 104 | + applicable therein; disputes belong to the courts of Québec, Canada. | |
| 105 | + </p> | |
| 106 | + </section> | |
| 107 | + </div> | |
| 108 | + ); | |
| 109 | +} | |
added
apps/web/components/EfficiencyLive.tsx
+143 −0
@@ -0,0 +1,143 @@ | ||
| 1 | +/** | |
| 2 | + * llmindex.io — live-updating efficiency (Pareto) frontier table | |
| 3 | + * Author: Simon-Pierre Boucher | |
| 4 | + * Contact: contact@spboucher.ai | |
| 5 | + * License: Proprietary — © Simon-Pierre Boucher, all rights reserved | |
| 6 | + */ | |
| 7 | +'use client'; | |
| 8 | + | |
| 9 | +import { useEffect, useState } from 'react'; | |
| 10 | +import Link from 'next/link'; | |
| 11 | +import { formatUsd } from '@llmindex/ui'; | |
| 12 | +import { ProviderLogo } from './ProviderLogo'; | |
| 13 | + | |
| 14 | +export interface EffPoint { | |
| 15 | + model: string; | |
| 16 | + name: string; | |
| 17 | + score: number; | |
| 18 | + cost_per_1k_items: number; | |
| 19 | + on_frontier: boolean; | |
| 20 | +} | |
| 21 | + | |
| 22 | +const POLL_MS = 15000; | |
| 23 | + | |
| 24 | +export function EfficiencyLive({ initial }: { initial: EffPoint[] }) { | |
| 25 | + const [points, setPoints] = useState<EffPoint[]>(initial); | |
| 26 | + | |
| 27 | + useEffect(() => { | |
| 28 | + let cancelled = false; | |
| 29 | + async function tick() { | |
| 30 | + try { | |
| 31 | + const res = await fetch('/api/v1/efficiency', { cache: 'no-store' }); | |
| 32 | + if (!res.ok || cancelled) return; | |
| 33 | + const body = (await res.json()) as { points: EffPoint[] }; | |
| 34 | + if (body.points) setPoints(body.points); | |
| 35 | + } catch { | |
| 36 | + /* keep last state */ | |
| 37 | + } | |
| 38 | + } | |
| 39 | + const id = setInterval(tick, POLL_MS); | |
| 40 | + tick(); | |
| 41 | + return () => { | |
| 42 | + cancelled = true; | |
| 43 | + clearInterval(id); | |
| 44 | + }; | |
| 45 | + }, []); | |
| 46 | + | |
| 47 | + if (points.length === 0) return null; | |
| 48 | + | |
| 49 | + return ( | |
| 50 | + <section className="space-y-3"> | |
| 51 | + <h2 className="text-lg font-semibold text-zinc-900 sm:text-xl">Efficiency frontier</h2> | |
| 52 | + <p className="text-sm text-zinc-500"> | |
| 53 | + Score vs. measured cost per 1k items — refreshed live as models land. Frontier models are | |
| 54 | + not dominated on both axes; never collapsed into a single blended number. | |
| 55 | + </p> | |
| 56 | + <div className="overflow-x-auto rounded-xl border border-zinc-200 bg-white"> | |
| 57 | + <table className="w-full min-w-[28rem] text-sm"> | |
| 58 | + <thead> | |
| 59 | + <tr className="border-b border-zinc-200 text-left text-xs uppercase tracking-wide text-zinc-500"> | |
| 60 | + <th className="px-3 py-2">Model</th> | |
| 61 | + <th className="px-3 py-2 text-right">Global Index</th> | |
| 62 | + <th className="px-3 py-2 text-right">Cost / 1k items</th> | |
| 63 | + <th className="px-3 py-2">Pareto</th> | |
| 64 | + </tr> | |
| 65 | + </thead> | |
| 66 | + <tbody> | |
| 67 | + {points.map((p) => ( | |
| 68 | + <tr key={p.model} className="border-b border-zinc-100 last:border-0"> | |
| 69 | + <td className="px-3 py-2"> | |
| 70 | + <Link href={`/models/${p.model}`} className="flex items-center gap-2 hover:text-emerald-600"> | |
| 71 | + <ProviderLogo provider={p.model.split('/')[0] ?? ''} size={16} /> | |
| 72 | + {p.name} | |
| 73 | + </Link> | |
| 74 | + </td> | |
| 75 | + <td className="px-3 py-2 text-right tabular-nums">{p.score}</td> | |
| 76 | + <td className="px-3 py-2 text-right tabular-nums">{formatUsd(p.cost_per_1k_items)}</td> | |
| 77 | + <td className="px-3 py-2"> | |
| 78 | + {p.on_frontier ? ( | |
| 79 | + <span className="rounded-full bg-emerald-100 px-2 py-0.5 text-xs text-emerald-700"> | |
| 80 | + frontier | |
| 81 | + </span> | |
| 82 | + ) : ( | |
| 83 | + <span className="text-xs text-zinc-400">dominated</span> | |
| 84 | + )} | |
| 85 | + </td> | |
| 86 | + </tr> | |
| 87 | + ))} | |
| 88 | + </tbody> | |
| 89 | + </table> | |
| 90 | + </div> | |
| 91 | + </section> | |
| 92 | + ); | |
| 93 | +} | |
| 94 | + | |
| 95 | +export function HomeStats({ | |
| 96 | + initialCount, | |
| 97 | + domains, | |
| 98 | + version, | |
| 99 | +}: { | |
| 100 | + initialCount: number; | |
| 101 | + domains: number; | |
| 102 | + version: string; | |
| 103 | +}) { | |
| 104 | + const [count, setCount] = useState(initialCount); | |
| 105 | + | |
| 106 | + useEffect(() => { | |
| 107 | + let cancelled = false; | |
| 108 | + async function tick() { | |
| 109 | + try { | |
| 110 | + const res = await fetch('/api/v1/leaderboard?limit=200', { cache: 'no-store' }); | |
| 111 | + if (!res.ok || cancelled) return; | |
| 112 | + const body = (await res.json()) as { entries?: unknown[] }; | |
| 113 | + if (body.entries) setCount(body.entries.length); | |
| 114 | + } catch { | |
| 115 | + /* keep last state */ | |
| 116 | + } | |
| 117 | + } | |
| 118 | + const id = setInterval(tick, POLL_MS); | |
| 119 | + return () => { | |
| 120 | + cancelled = true; | |
| 121 | + clearInterval(id); | |
| 122 | + }; | |
| 123 | + }, []); | |
| 124 | + | |
| 125 | + return ( | |
| 126 | + <div className="flex flex-wrap gap-x-6 gap-y-2 pt-2 text-sm"> | |
| 127 | + <span className="flex items-center gap-2 text-zinc-700"> | |
| 128 | + <span className="relative flex h-2 w-2"> | |
| 129 | + <span className="absolute inline-flex h-full w-full animate-ping rounded-full bg-emerald-400 opacity-60" /> | |
| 130 | + <span className="relative inline-flex h-2 w-2 rounded-full bg-emerald-400" /> | |
| 131 | + </span> | |
| 132 | + updates live | |
| 133 | + </span> | |
| 134 | + <span className="text-zinc-600"> | |
| 135 | + <span className="font-semibold text-zinc-900 tabular-nums">{count}</span> models scored | |
| 136 | + </span> | |
| 137 | + <span className="text-zinc-600"> | |
| 138 | + <span className="font-semibold text-zinc-900 tabular-nums">{domains}</span> domains | |
| 139 | + </span> | |
| 140 | + <span className="text-zinc-600">IRT 2PL · 95% CI · v{version}</span> | |
| 141 | + </div> | |
| 142 | + ); | |
| 143 | +} | |
modified
apps/web/lib/data.ts
+11 −3
@@ -27,16 +27,24 @@ export interface LeaderboardData { | ||
| 27 | 27 | entries: LeaderboardEntry[]; |
| 28 | 28 | } |
| 29 | 29 | |
| 30 | −/** Latest displayable run: prefer a real index fit over the demo seed. */ | |
| 30 | +/** | |
| 31 | + * Latest displayable run — CURRENT index version only. Older-version runs | |
| 32 | + * stay in the database as immutable audit history but never mix into the | |
| 33 | + * displayed rankings. | |
| 34 | + */ | |
| 31 | 35 | export async function getLatestRun(): Promise<ScoreRun | null> { |
| 32 | 36 | try { |
| 33 | 37 | const fit = await prisma.scoreRun.findFirst({ |
| 34 | − where: { kind: 'index_fit', status: { in: ['complete', 'degraded'] } }, | |
| 38 | + where: { | |
| 39 | + kind: 'index_fit', | |
| 40 | + indexVersion: INDEX_VERSION, | |
| 41 | + status: { in: ['complete', 'degraded'] }, | |
| 42 | + }, | |
| 35 | 43 | orderBy: { createdAt: 'desc' }, |
| 36 | 44 | }); |
| 37 | 45 | if (fit) return fit; |
| 38 | 46 | return await prisma.scoreRun.findFirst({ |
| 39 | − where: { kind: 'demo_seed', status: 'complete' }, | |
| 47 | + where: { kind: 'demo_seed', indexVersion: INDEX_VERSION, status: 'complete' }, | |
| 40 | 48 | orderBy: { createdAt: 'desc' }, |
| 41 | 49 | }); |
| 42 | 50 | } catch { |
| 43 | 51 | |