TypeScript 97.4%
SQL 1%
JavaScript 0.9%
CSS 0.6%
1/**2 * Arena scoring — pure, dependency-free helpers shared by the client (winner card, scoreboard view),3 * the API (scoreboard aggregation, exports, share snapshots) and the unit tests.4 *5 * Nothing in here touches the database or the network.6 */78// ---------------------------------------------------------------------------9// Task categories (from the Smart Router's prompt analysis)10// ---------------------------------------------------------------------------11export type TaskCategory = "coding" | "research" | "writing" | "reasoning" | "general";1213export const CATEGORIES: { value: TaskCategory; label: string }[] = [14 { value: "coding", label: "Coding" },15 { value: "research", label: "Research" },16 { value: "writing", label: "Writing" },17 { value: "reasoning", label: "Reasoning" },18 { value: "general", label: "General" },19];2021/** Map a router `TaskType` (or anything else) to the five scoreboard categories. */22export function categoryFromTask(task: string | null | undefined): TaskCategory {23 switch (task) {24 case "coding":25 return "coding";26 case "research":27 return "research";28 case "writing":29 case "translation":30 return "writing";31 case "reasoning":32 case "analysis":33 return "reasoning";34 default:35 return "general";36 }37}3839export function isTaskCategory(v: unknown): v is TaskCategory {40 return typeof v === "string" && CATEGORIES.some((c) => c.value === v);41}4243// ---------------------------------------------------------------------------44// Criteria45// ---------------------------------------------------------------------------46export interface Criterion {47 id: string;48 label: string;49 short: string;50 /** Legacy `arena_responses.ratings` key kept in sync for backward compatibility. */51 ratingKey: string;52 custom?: boolean;53}5455export const BUILTIN_CRITERIA: Criterion[] = [56 { id: "best", label: "Best answer", short: "Best", ratingKey: "best" },57 { id: "accurate", label: "Most accurate", short: "Accurate", ratingKey: "bestAccurate" },58 { id: "writing", label: "Best writing", short: "Writing", ratingKey: "bestWriting" },59 { id: "coding", label: "Best coding", short: "Coding", ratingKey: "bestCoding" },60 { id: "value", label: "Best value", short: "Value", ratingKey: "bestValue" },61 { id: "fastest", label: "Fastest", short: "Fastest", ratingKey: "bestSpeed" },62];6364export const CUSTOM_CRITERION_PREFIX = "custom:";65export const CRITERION_RE = /^(best|accurate|writing|coding|value|fastest|custom:[a-z0-9][a-z0-9-]{0,31})$/;6667export function slugifyCriterion(label: string): string {68 return label69 .normalize("NFD")70 .replace(/[̀-ͯ]/g, "")71 .toLowerCase()72 .replace(/[^a-z0-9]+/g, "-")73 .replace(/^-+|-+$/g, "")74 .slice(0, 32);75}7677/** Build a custom criterion from a user-typed label; null when the label is unusable. */78export function customCriterion(label: string): Criterion | null {79 const clean = label.trim().slice(0, 40);80 const slug = slugifyCriterion(clean);81 if (!slug) return null;82 const id = `${CUSTOM_CRITERION_PREFIX}${slug}`;83 return { id, label: clean, short: clean, ratingKey: `bestCustom_${slug}`.slice(0, 32), custom: true };84}8586export function isCustomCriterion(id: string): boolean {87 return id.startsWith(CUSTOM_CRITERION_PREFIX);88}8990/** Resolve a criterion id to its definition (custom ids fall back to a humanised slug). */91export function criterionById(id: string, custom: Criterion[] = []): Criterion {92 const found = BUILTIN_CRITERIA.find((c) => c.id === id) ?? custom.find((c) => c.id === id);93 if (found) return found;94 const slug = id.startsWith(CUSTOM_CRITERION_PREFIX) ? id.slice(CUSTOM_CRITERION_PREFIX.length) : id;95 const label = slug.replace(/-/g, " ").replace(/^\w/, (c) => c.toUpperCase());96 return { id, label, short: label, ratingKey: `bestCustom_${slug}`.slice(0, 32), custom: true };97}9899/** Rating key written to `arena_responses.ratings` for a criterion. */100export function ratingKeyFor(criterion: string): string {101 return criterionById(criterion).ratingKey;102}103104// ---------------------------------------------------------------------------105// Winner106// ---------------------------------------------------------------------------107export interface ResponseLike {108 id: string;109 modelKey: string;110 status: string;111 ttftMs: number | null;112 latencyMs: number | null;113 costUsd: number | null;114 usage: { inputTokens?: number; outputTokens?: number } | null;115}116117export interface VoteLike {118 criterion: string;119 responseId: string;120 modelKey: string;121}122123export interface WinnerDeltas {124 /** Winner − mean of the other responses (negative = winner is cheaper / faster / shorter). */125 costUsd: number | null;126 ttftMs: number | null;127 latencyMs: number | null;128 outputTokens: number | null;129 /** Number of other responses the deltas were computed against. */130 others: number;131}132133export interface WinnerResult {134 modelKey: string;135 responseId: string;136 criteriaWon: string[];137 votes: number;138 /** How the tie was broken (null = clear majority). */139 tieBreak: "fastest" | "order" | null;140 deltas: WinnerDeltas;141}142143function mean(values: (number | null | undefined)[]): number | null {144 const v = values.filter((x): x is number => typeof x === "number" && Number.isFinite(x));145 return v.length ? v.reduce((a, b) => a + b, 0) / v.length : null;146}147148/**149 * Arena Winner: the response that won the most criteria; ties go to the fastest time to first token150 * (then total latency, then original order). Returns null when there is no vote at all.151 */152export function computeWinner(responses: ResponseLike[], votes: VoteLike[]): WinnerResult | null {153 if (!responses.length || !votes.length) return null;154 const byResponse = new Map<string, string[]>();155 for (const v of votes) {156 const r = responses.find((x) => x.id === v.responseId) ?? responses.find((x) => x.modelKey === v.modelKey);157 if (!r) continue;158 const list = byResponse.get(r.id) ?? [];159 if (!list.includes(v.criterion)) list.push(v.criterion);160 byResponse.set(r.id, list);161 }162 if (!byResponse.size) return null;163 const ranked = responses164 .filter((r) => byResponse.has(r.id))165 .map((r) => ({ r, won: byResponse.get(r.id)! }))166 .sort((a, b) => b.won.length - a.won.length);167 const top = ranked.filter((x) => x.won.length === ranked[0].won.length);168 let tieBreak: WinnerResult["tieBreak"] = null;169 let winner = top[0];170 if (top.length > 1) {171 const speed = (r: ResponseLike) => r.ttftMs ?? r.latencyMs ?? Number.POSITIVE_INFINITY;172 const sorted = [...top].sort((a, b) => speed(a.r) - speed(b.r));173 winner = sorted[0];174 tieBreak = Number.isFinite(speed(winner.r)) && speed(sorted[0].r) !== speed(sorted[1].r) ? "fastest" : "order";175 }176 const others = responses.filter((r) => r.id !== winner.r.id && r.status === "complete");177 const w = winner.r;178 const delta = (get: (r: ResponseLike) => number | null | undefined): number | null => {179 const mine = get(w);180 const avg = mean(others.map(get));181 return typeof mine === "number" && avg !== null ? mine - avg : null;182 };183 return {184 modelKey: w.modelKey,185 responseId: w.id,186 criteriaWon: winner.won,187 votes: winner.won.length,188 tieBreak,189 deltas: {190 costUsd: delta((r) => r.costUsd),191 ttftMs: delta((r) => r.ttftMs),192 latencyMs: delta((r) => r.latencyMs),193 outputTokens: delta((r) => r.usage?.outputTokens),194 others: others.length,195 },196 };197}198199// ---------------------------------------------------------------------------200// Scoreboard201// ---------------------------------------------------------------------------202export interface ScoreboardSession {203 id: string;204 modelKeys: string[];205 category: TaskCategory;206}207208export interface ScoreboardVote extends VoteLike {209 sessionId: string;210}211212export interface ScoreboardResponse extends ResponseLike {213 sessionId: string;214}215216export interface ScoreboardFilter {217 category?: TaskCategory | null;218 /** Criterion id (e.g. `value` for cost efficiency). */219 criterion?: string | null;220}221222export interface ScoreboardRow {223 modelKey: string;224 provider: string;225 /** Sessions the model took part in (after filtering). */226 sessions: number;227 /** Sessions with at least one qualifying vote — the denominator of the win rate. */228 decided: number;229 /** Sessions the model won (Arena Winner on the filtered criteria). */230 wins: number;231 /** Individual criteria won across sessions. */232 votes: number;233 /** wins / decided, 0..1. */234 winRate: number;235 avgCostUsd: number | null;236 avgTtftMs: number | null;237 criteria: Record<string, number>;238}239240/**241 * Personal scoreboard: for each model, the share of *decided* comparisons it won.242 * A comparison is decided when it has at least one vote matching the filter; undecided243 * sessions still count as participation but cannot be won by anyone.244 */245export function computeScoreboard(input: { sessions: ScoreboardSession[]; responses: ScoreboardResponse[]; votes: ScoreboardVote[] }, filter: ScoreboardFilter = {}): ScoreboardRow[] {246 const sessions = filter.category ? input.sessions.filter((s) => s.category === filter.category) : input.sessions;247 const sessionIds = new Set(sessions.map((s) => s.id));248 const votes = input.votes.filter((v) => sessionIds.has(v.sessionId) && (!filter.criterion || v.criterion === filter.criterion));249 const responses = input.responses.filter((r) => sessionIds.has(r.sessionId));250251 const rows = new Map<string, ScoreboardRow & { costs: number[]; ttfts: number[] }>();252 const row = (key: string) => {253 let r = rows.get(key);254 if (!r) {255 r = { modelKey: key, provider: key.split("/")[0] ?? "", sessions: 0, decided: 0, wins: 0, votes: 0, winRate: 0, avgCostUsd: null, avgTtftMs: null, criteria: {}, costs: [], ttfts: [] };256 rows.set(key, r);257 }258 return r;259 };260261 for (const s of sessions) {262 const sVotes = votes.filter((v) => v.sessionId === s.id);263 const sResponses = responses.filter((r) => r.sessionId === s.id);264 const participants = new Set([...s.modelKeys, ...sResponses.map((r) => r.modelKey)]);265 for (const k of participants) {266 const r = row(k);267 r.sessions += 1;268 if (sVotes.length) r.decided += 1;269 }270 for (const resp of sResponses) {271 if (resp.status !== "complete") continue;272 const r = row(resp.modelKey);273 if (typeof resp.costUsd === "number") r.costs.push(resp.costUsd);274 if (typeof resp.ttftMs === "number") r.ttfts.push(resp.ttftMs);275 }276 for (const v of sVotes) {277 const r = row(v.modelKey);278 r.votes += 1;279 r.criteria[v.criterion] = (r.criteria[v.criterion] ?? 0) + 1;280 }281 const winner = computeWinner(sResponses.length ? sResponses : s.modelKeys.map((k) => ({ id: k, modelKey: k, status: "complete", ttftMs: null, latencyMs: null, costUsd: null, usage: null })), sVotes);282 if (winner) row(winner.modelKey).wins += 1;283 }284285 return [...rows.values()]286 .map(({ costs, ttfts, ...r }) => ({ ...r, winRate: r.decided ? r.wins / r.decided : 0, avgCostUsd: mean(costs), avgTtftMs: mean(ttfts) }))287 .sort((a, b) => b.winRate - a.winRate || b.wins - a.wins || b.votes - a.votes || b.sessions - a.sessions || a.modelKey.localeCompare(b.modelKey));288}289290// ---------------------------------------------------------------------------291// Blind Arena292// ---------------------------------------------------------------------------293export const BLIND_LETTERS = ["A", "B", "C", "D"] as const;294295export function blindLabel(index: number): string {296 return `Model ${BLIND_LETTERS[index] ?? String(index + 1)}`;297}298299/** Fisher–Yates permutation of `0..n-1` (used to hide selection order in Blind Arena). */300export function shuffledOrder(n: number, random: () => number = Math.random): number[] {301 const order = Array.from({ length: n }, (_, i) => i);302 for (let i = n - 1; i > 0; i--) {303 const j = Math.floor(random() * (i + 1));304 [order[i], order[j]] = [order[j], order[i]];305 }306 return order;307}308309/** Validate a stored permutation against a model count; falls back to identity. */310export function normalizeOrder(order: unknown, n: number): number[] {311 const identity = Array.from({ length: n }, (_, i) => i);312 if (!Array.isArray(order) || order.length !== n) return identity;313 const seen = new Set<number>();314 for (const v of order) {315 if (typeof v !== "number" || !Number.isInteger(v) || v < 0 || v >= n || seen.has(v)) return identity;316 seen.add(v);317 }318 return order as number[];319}320