TypeScript 97.4%
SQL 1%
JavaScript 0.9%
CSS 0.6%
1import { describe, it, expect } from "vitest";2import { computeWinner, computeScoreboard, categoryFromTask, customCriterion, criterionById, ratingKeyFor, shuffledOrder, normalizeOrder, blindLabel, CRITERION_RE, type ResponseLike } from "@/lib/arena/scoring";3import { liveMetrics, formatDelta } from "@/lib/arena/metrics";4import { buildArenaMarkdown, buildArenaJson, buildArenaShareSnapshot, generationParameters, arenaExportFilename } from "@/lib/arena/export";56const r = (id: string, modelKey: string, over: Partial<ResponseLike> = {}): ResponseLike => ({ id, modelKey, status: "complete", ttftMs: 500, latencyMs: 3000, costUsd: 0.01, usage: { inputTokens: 100, outputTokens: 400 }, ...over });78describe("computeWinner", () => {9 const responses = [r("a", "openai/gpt", { ttftMs: 800, costUsd: 0.02, usage: { inputTokens: 100, outputTokens: 500 } }), r("b", "anthropic/claude", { ttftMs: 400, costUsd: 0.01, usage: { inputTokens: 100, outputTokens: 300 } }), r("c", "gemini/flash", { ttftMs: 300, costUsd: 0.002, usage: { inputTokens: 100, outputTokens: 200 } })];1011 it("returns null without votes", () => {12 expect(computeWinner(responses, [])).toBeNull();13 expect(computeWinner([], [{ criterion: "best", responseId: "a", modelKey: "openai/gpt" }])).toBeNull();14 });1516 it("picks the response with the most criteria", () => {17 const w = computeWinner(responses, [18 { criterion: "best", responseId: "a", modelKey: "openai/gpt" },19 { criterion: "writing", responseId: "a", modelKey: "openai/gpt" },20 { criterion: "fastest", responseId: "c", modelKey: "gemini/flash" },21 ]);22 expect(w?.modelKey).toBe("openai/gpt");23 expect(w?.criteriaWon.sort()).toEqual(["best", "writing"]);24 expect(w?.tieBreak).toBeNull();25 });2627 it("breaks ties by time to first token", () => {28 const w = computeWinner(responses, [29 { criterion: "best", responseId: "a", modelKey: "openai/gpt" },30 { criterion: "coding", responseId: "b", modelKey: "anthropic/claude" },31 ]);32 expect(w?.modelKey).toBe("anthropic/claude");33 expect(w?.tieBreak).toBe("fastest");34 });3536 it("computes deltas against the mean of the others", () => {37 const w = computeWinner(responses, [{ criterion: "best", responseId: "c", modelKey: "gemini/flash" }])!;38 expect(w.deltas.others).toBe(2);39 expect(w.deltas.costUsd).toBeCloseTo(0.002 - 0.015, 6);40 expect(w.deltas.ttftMs).toBeCloseTo(300 - 600);41 expect(w.deltas.outputTokens).toBeCloseTo(200 - 400);42 });4344 it("ignores votes pointing at unknown responses and dedupes criteria", () => {45 const w = computeWinner(responses, [46 { criterion: "best", responseId: "zzz", modelKey: "nope/none" },47 { criterion: "value", responseId: "b", modelKey: "anthropic/claude" },48 { criterion: "value", responseId: "b", modelKey: "anthropic/claude" },49 ]);50 expect(w?.modelKey).toBe("anthropic/claude");51 expect(w?.votes).toBe(1);52 });53});5455describe("computeScoreboard", () => {56 const sessions = [57 { id: "s1", modelKeys: ["openai/gpt", "anthropic/claude"], category: "coding" as const },58 { id: "s2", modelKeys: ["openai/gpt", "anthropic/claude"], category: "writing" as const },59 { id: "s3", modelKeys: ["openai/gpt", "gemini/flash"], category: "coding" as const },60 ];61 const responses = [62 { ...r("a1", "openai/gpt", { costUsd: 0.02, ttftMs: 800 }), sessionId: "s1" },63 { ...r("b1", "anthropic/claude", { costUsd: 0.01, ttftMs: 400 }), sessionId: "s1" },64 { ...r("a2", "openai/gpt", { costUsd: 0.04, ttftMs: 600 }), sessionId: "s2" },65 { ...r("b2", "anthropic/claude", { costUsd: 0.01, ttftMs: 500 }), sessionId: "s2" },66 { ...r("a3", "openai/gpt", { costUsd: 0.03, ttftMs: 700 }), sessionId: "s3" },67 { ...r("c3", "gemini/flash", { costUsd: 0.001, ttftMs: 200 }), sessionId: "s3" },68 ];69 const votes = [70 { sessionId: "s1", criterion: "best", responseId: "b1", modelKey: "anthropic/claude" },71 { sessionId: "s1", criterion: "coding", responseId: "b1", modelKey: "anthropic/claude" },72 { sessionId: "s2", criterion: "best", responseId: "a2", modelKey: "openai/gpt" },73 { sessionId: "s2", criterion: "value", responseId: "b2", modelKey: "anthropic/claude" },74 // s3 undecided75 ];7677 it("ranks by win rate over decided sessions", () => {78 const rows = computeScoreboard({ sessions, responses, votes });79 const claude = rows.find((x) => x.modelKey === "anthropic/claude")!;80 const gpt = rows.find((x) => x.modelKey === "openai/gpt")!;81 const flash = rows.find((x) => x.modelKey === "gemini/flash")!;82 expect(claude.sessions).toBe(2);83 expect(claude.decided).toBe(2);84 expect(claude.wins).toBe(2); // s1 (2 criteria) + s2 (1–1 tie → Claude's 500 ms TTFT beats 600 ms)85 expect(claude.winRate).toBe(1);86 expect(gpt.sessions).toBe(3);87 expect(gpt.decided).toBe(2);88 expect(gpt.wins).toBe(0);89 expect(flash.decided).toBe(0);90 expect(flash.winRate).toBe(0);91 expect(rows[0].modelKey).toBe("anthropic/claude");92 expect(claude.votes).toBe(3);93 expect(claude.avgCostUsd).toBeCloseTo(0.01);94 expect(gpt.avgTtftMs).toBeCloseTo(700);95 });9697 it("s2 tie goes to the faster model", () => {98 const rows = computeScoreboard({ sessions: [sessions[1]], responses, votes });99 // a2 ttft 600 vs b2 ttft 500 → claude wins the tie100 expect(rows.find((x) => x.modelKey === "anthropic/claude")!.wins).toBe(1);101 expect(rows.find((x) => x.modelKey === "openai/gpt")!.wins).toBe(0);102 });103104 it("filters by category", () => {105 const rows = computeScoreboard({ sessions, responses, votes }, { category: "coding" });106 expect(rows.find((x) => x.modelKey === "anthropic/claude")!.sessions).toBe(1);107 expect(rows.find((x) => x.modelKey === "openai/gpt")!.sessions).toBe(2);108 expect(rows.find((x) => x.modelKey === "openai/gpt")!.decided).toBe(1);109 });110111 it("filters by criterion (cost efficiency = value)", () => {112 const rows = computeScoreboard({ sessions, responses, votes }, { criterion: "value" });113 const claude = rows.find((x) => x.modelKey === "anthropic/claude")!;114 expect(claude.wins).toBe(1);115 expect(claude.decided).toBe(1);116 expect(claude.winRate).toBe(1);117 expect(rows.find((x) => x.modelKey === "openai/gpt")!.wins).toBe(0);118 });119120 it("returns an empty list without sessions", () => {121 expect(computeScoreboard({ sessions: [], responses: [], votes: [] })).toEqual([]);122 });123});124125describe("criteria & categories", () => {126 it("maps router tasks to categories", () => {127 expect(categoryFromTask("coding")).toBe("coding");128 expect(categoryFromTask("analysis")).toBe("reasoning");129 expect(categoryFromTask("translation")).toBe("writing");130 expect(categoryFromTask("chat")).toBe("general");131 expect(categoryFromTask(undefined)).toBe("general");132 });133 it("builds custom criteria with stable ids", () => {134 const c = customCriterion(" Most Concise! ")!;135 expect(c.id).toBe("custom:most-concise");136 expect(c.label).toBe("Most Concise!");137 expect(CRITERION_RE.test(c.id)).toBe(true);138 expect(customCriterion("!!!")).toBeNull();139 expect(criterionById("custom:most-concise").label).toBe("Most concise");140 });141 it("keeps legacy rating keys", () => {142 expect(ratingKeyFor("best")).toBe("best");143 expect(ratingKeyFor("fastest")).toBe("bestSpeed");144 expect(ratingKeyFor("coding")).toBe("bestCoding");145 expect(ratingKeyFor("custom:tone").startsWith("bestCustom_")).toBe(true);146 });147});148149describe("blind helpers", () => {150 it("shuffles into a valid permutation", () => {151 const o = shuffledOrder(4, () => 0.42);152 expect([...o].sort()).toEqual([0, 1, 2, 3]);153 expect(normalizeOrder(o, 4)).toEqual(o);154 expect(normalizeOrder([0, 0, 1], 3)).toEqual([0, 1, 2]);155 expect(normalizeOrder("nope", 2)).toEqual([0, 1]);156 expect(blindLabel(2)).toBe("Model C");157 });158});159160describe("liveMetrics", () => {161 const model = { key: "openai/gpt", id: "gpt", provider: "openai" as const, displayName: "GPT", capabilities: { text: true, vision: false, audioInput: false, audioOutput: false, imageGeneration: false, video: false, reasoning: false, tools: false, structuredOutput: false, streaming: true, files: false, webSearch: false }, limits: {}, parameters: {}, status: "active" as const, pricing: { inputPerMillion: 1, outputPerMillion: 10 } };162 it("estimates while streaming and switches to exact when done", () => {163 const t0 = 1_000;164 const live = liveMetrics({ status: "streaming", text: "x".repeat(400), startedAt: t0, firstTokenAt: t0 + 500, response: null }, model, 200, t0 + 2500);165 expect(live.exact).toBe(false);166 expect(live.ttftMs).toBe(500);167 expect(live.outputTokens).toBe(100);168 expect(live.tokensPerSecond).toBe(50);169 expect(live.costUsd).toBeCloseTo(200 / 1e6 + 100 * 10 / 1e6, 9);170 const done = liveMetrics({ status: "done", text: "x", startedAt: t0, firstTokenAt: t0 + 500, response: { ttftMs: 480, latencyMs: 2480, costUsd: 0.0012, usage: { inputTokens: 210, outputTokens: 120 } } }, model, 200, t0 + 9999);171 expect(done.exact).toBe(true);172 expect(done.ttftMs).toBe(480);173 expect(done.outputTokens).toBe(120);174 expect(done.tokensPerSecond).toBe(60);175 expect(done.costUsd).toBe(0.0012);176 });177 it("formats signed deltas", () => {178 expect(formatDelta(-0.0031, "usd")).toBe("−$0.0031");179 expect(formatDelta(120, "ms")).toBe("+120 ms");180 expect(formatDelta(-340, "tokens")).toBe("−340 tok");181 expect(formatDelta(null, "ms")).toBe("—");182 });183});184185describe("exports", () => {186 const input = {187 session: { id: "arn_1", prompt: "Explain monads simply", systemPrompt: "Be brief", modelKeys: ["openai/gpt", "anthropic/claude"], settings: { temperature: 0.3, attachmentIds: ["att_1"], blind: true, blindOrder: [1, 0] }, createdAt: "2026-09-11T12:00:00.000Z" },188 responses: [189 { ...r("a", "openai/gpt"), provider: "openai", content: "A monad is…", reasoning: null, error: null, ratings: { best: false }, createdAt: "2026-09-11T12:00:01.000Z" },190 { ...r("b", "anthropic/claude", { ttftMs: 300 }), provider: "anthropic", content: "Think of a box…", reasoning: "hmm", error: null, ratings: { best: true }, createdAt: "2026-09-11T12:00:01.000Z" },191 ],192 votes: [{ criterion: "best", responseId: "b", modelKey: "anthropic/claude", category: "general", createdAt: "2026-09-11T12:00:05.000Z" }],193 models: { "openai/gpt": { displayName: "GPT", provider: "openai" }, "anthropic/claude": { displayName: "Claude", provider: "anthropic" } },194 };195 it("strips Arena flags from parameters", () => {196 expect(generationParameters(input.session.settings)).toEqual({ temperature: 0.3 });197 expect(arenaExportFilename("Explain monads simply", "md")).toBe("arena-explain-monads-simply.md");198 });199 it("builds markdown with prompt, metrics, votes, winner and responses", () => {200 const md = buildArenaMarkdown(input);201 expect(md).toContain("# Arena comparison — 2 models");202 expect(md).toContain("Blind Arena");203 expect(md).toContain("Explain monads simply");204 expect(md).toContain("| Claude | complete | 300 ms |");205 expect(md).toContain("**Best answer** → Claude");206 expect(md).toContain("**Arena Winner: Claude**");207 expect(md).toContain("1 attachment (not included");208 expect(md).toContain("### Claude (anthropic)");209 expect(md).toContain("<details><summary>Reasoning</summary>");210 });211 it("builds JSON and a share snapshot without attachment data", () => {212 const json = JSON.parse(buildArenaJson(input));213 expect(json.kind).toBe("arena-session");214 expect(json.session.parameters).toEqual({ temperature: 0.3 });215 expect(json.session.attachmentCount).toBe(1);216 expect(json.winner.displayName).toBe("Claude");217 expect(json.responses[1].criteriaWon).toEqual(["best"]);218 const snap = buildArenaShareSnapshot(input);219 expect(snap.version).toBe(1);220 expect(snap.blind).toBe(true);221 expect(snap.models.map((m) => m.displayName)).toEqual(["GPT", "Claude"]);222 expect(snap.winner?.modelKey).toBe("anthropic/claude");223 expect(JSON.stringify(snap)).not.toContain("att_1");224 expect(JSON.stringify(snap)).not.toContain("dataBase64");225 });226});227