import { computeWinner, criterionById, type ResponseLike, type VoteLike } from "./scoring"; /** * Arena exports (Markdown / JSON) and the public share snapshot. Pure functions over plain data so the * API route, the share page and the unit tests share one implementation. */ export interface ExportModelInfo { displayName: string; provider: string; } export interface ArenaExportResponse extends ResponseLike { provider: string; content: string; reasoning: string | null; error: { code: string; message: string } | null; ratings: Record; createdAt: string; } export interface ArenaExportVote extends VoteLike { category: string | null; createdAt: string; } export interface ArenaExportInput { session: { id: string; prompt: string; systemPrompt: string | null; modelKeys: string[]; settings: Record; createdAt: string; }; responses: ArenaExportResponse[]; votes: ArenaExportVote[]; models: Record; } /** Generation parameters only (the session settings also carry attachment ids and Arena flags). */ export function generationParameters(settings: Record): Record { const { attachmentIds: _a, blind: _b, blindOrder: _o, tools: _t, ...rest } = settings ?? {}; void _a; void _b; void _o; void _t; return rest; } export function attachmentCount(settings: Record): number { const ids = (settings as { attachmentIds?: unknown })?.attachmentIds; return Array.isArray(ids) ? ids.length : 0; } export function isBlind(settings: Record): boolean { return (settings as { blind?: unknown })?.blind === true; } export function modelName(key: string, models: Record): string { return models[key]?.displayName ?? key.split("/").slice(1).join("/") ?? key; } function tokensPerSecond(r: ResponseLike): number | null { const out = r.usage?.outputTokens; if (!out || r.latencyMs === null) return null; const gen = Math.max(1, r.latencyMs - (r.ttftMs ?? 0)); return Math.round((out / gen) * 1000); } function fmtMs(ms: number | null | undefined): string { if (ms === null || ms === undefined) return "—"; return ms < 1000 ? `${Math.round(ms)} ms` : `${(ms / 1000).toFixed(2)} s`; } function fmtUsd(v: number | null | undefined): string { if (v === null || v === undefined) return "—"; return v < 0.01 ? `$${v.toFixed(v < 0.001 ? 5 : 4)}` : `$${v.toFixed(2)}`; } function fmtInt(v: number | null | undefined): string { return typeof v === "number" ? new Intl.NumberFormat("en-US").format(v) : "—"; } export function arenaExportFilename(prompt: string, ext: "md" | "json"): string { const slug = prompt .normalize("NFD") .replace(/[̀-ͯ]/g, "") .toLowerCase() .replace(/[^a-z0-9]+/g, "-") .replace(/^-+|-+$/g, "") .slice(0, 48) || "arena"; return `arena-${slug}.${ext}`; } export function buildArenaJson(input: ArenaExportInput): string { const { session, responses, votes, models } = input; const winner = computeWinner(responses, votes); return JSON.stringify( { exportedAt: new Date().toISOString(), app: "PolyLLM", kind: "arena-session", version: 1, session: { id: session.id, createdAt: session.createdAt, prompt: session.prompt, systemPrompt: session.systemPrompt, blind: isBlind(session.settings), attachmentCount: attachmentCount(session.settings), parameters: generationParameters(session.settings), }, models: session.modelKeys.map((key) => ({ key, provider: models[key]?.provider ?? key.split("/")[0], displayName: modelName(key, models) })), responses: responses.map((r) => ({ id: r.id, modelKey: r.modelKey, provider: r.provider, displayName: modelName(r.modelKey, models), status: r.status, content: r.content, reasoning: r.reasoning, error: r.error, metrics: { ttftMs: r.ttftMs, latencyMs: r.latencyMs, tokensPerSecond: tokensPerSecond(r), usage: r.usage, costUsd: r.costUsd }, ratings: r.ratings, criteriaWon: votes.filter((v) => v.responseId === r.id).map((v) => v.criterion), createdAt: r.createdAt, })), votes: votes.map((v) => ({ criterion: v.criterion, label: criterionById(v.criterion).label, responseId: v.responseId, modelKey: v.modelKey, category: v.category, createdAt: v.createdAt })), winner: winner ? { modelKey: winner.modelKey, displayName: modelName(winner.modelKey, models), criteriaWon: winner.criteriaWon, tieBreak: winner.tieBreak, deltas: winner.deltas } : null, }, null, 2, ); } export function buildArenaMarkdown(input: ArenaExportInput): string { const { session, responses, votes, models } = input; const winner = computeWinner(responses, votes); const params = generationParameters(session.settings); const lines: string[] = []; lines.push(`# Arena comparison — ${session.modelKeys.length} model${session.modelKeys.length === 1 ? "" : "s"}`, ""); lines.push(`_Exported from PolyLLM on ${new Date().toISOString().slice(0, 10)} · session created ${session.createdAt.slice(0, 16).replace("T", " ")} UTC_`, ""); if (isBlind(session.settings)) lines.push("> Blind Arena: model identities were hidden until the vote.", ""); lines.push("## Prompt", "", session.prompt, ""); if (session.systemPrompt) lines.push("**System prompt**", "", "> " + session.systemPrompt.replace(/\n/g, "\n> "), ""); const n = attachmentCount(session.settings); if (n) lines.push(`_${n} attachment${n === 1 ? "" : "s"} (not included in the export)_`, ""); lines.push("## Models & parameters", ""); lines.push("| Model | Provider |", "| --- | --- |"); for (const key of session.modelKeys) lines.push(`| ${modelName(key, models)} | ${models[key]?.provider ?? key.split("/")[0]} |`); lines.push(""); const entries = Object.entries(params).filter(([, v]) => v !== undefined && v !== null); if (entries.length) { lines.push("Shared generation settings (filtered per model — unsupported parameters are not sent):", ""); for (const [k, v] of entries) lines.push(`- \`${k}\`: ${typeof v === "object" ? JSON.stringify(v) : String(v)}`); lines.push(""); } else lines.push("Default generation settings.", ""); lines.push("## Metrics", ""); lines.push("| Model | Status | TTFT | Total | Tokens in → out | tok/s | Est. cost |", "| --- | --- | --- | --- | --- | --- | --- |"); for (const r of responses) { const tps = tokensPerSecond(r); lines.push(`| ${modelName(r.modelKey, models)} | ${r.status} | ${fmtMs(r.ttftMs)} | ${fmtMs(r.latencyMs)} | ${fmtInt(r.usage?.inputTokens)} → ${fmtInt(r.usage?.outputTokens)} | ${tps ?? "—"} | ${fmtUsd(r.costUsd)} |`); } lines.push(""); lines.push("## Votes", ""); if (!votes.length) lines.push("_No votes recorded._", ""); else { for (const v of votes) lines.push(`- **${criterionById(v.criterion).label}** → ${modelName(v.modelKey, models)}`); lines.push(""); if (winner) { lines.push(`**Arena Winner: ${modelName(winner.modelKey, models)}** — won ${winner.criteriaWon.length} criteri${winner.criteriaWon.length === 1 ? "on" : "a"}${winner.tieBreak === "fastest" ? " (tie broken by speed)" : ""}.`, ""); } } const legacy = responses.filter((r) => Object.values(r.ratings ?? {}).some(Boolean)); if (legacy.length) { lines.push("Legacy ratings:", ""); for (const r of legacy) lines.push(`- ${modelName(r.modelKey, models)}: ${Object.entries(r.ratings).filter(([, on]) => on).map(([k]) => k).join(", ")}`); lines.push(""); } lines.push("## Responses", ""); for (const r of responses) { lines.push(`### ${modelName(r.modelKey, models)} (${r.provider})`, ""); if (r.reasoning) lines.push("
Reasoning", "", r.reasoning, "", "
", ""); if (r.error) lines.push(`> **Error ${r.error.code}:** ${r.error.message}`, ""); lines.push(r.content || (r.status === "stopped" ? "_Stopped before any output._" : "_Empty response._"), ""); } return lines.join("\n"); } // --------------------------------------------------------------------------- // Public share snapshot // --------------------------------------------------------------------------- export interface ArenaShareSnapshot { version: 1; prompt: string; systemPrompt: string | null; blind: boolean; attachmentCount: number; parameters: Record; createdAt: string; models: { key: string; provider: string; displayName: string }[]; responses: { id: string; modelKey: string; provider: string; displayName: string; status: string; content: string; reasoning: string | null; error: { code: string; message: string } | null; ttftMs: number | null; latencyMs: number | null; costUsd: number | null; usage: { inputTokens?: number; outputTokens?: number; reasoningTokens?: number } | null; criteriaWon: string[]; }[]; votes: { criterion: string; label: string; modelKey: string; responseId: string }[]; winner: { modelKey: string; responseId: string; criteriaWon: string[]; tieBreak: "fastest" | "order" | null; deltas: { costUsd: number | null; ttftMs: number | null; latencyMs: number | null; outputTokens: number | null; others: number } } | null; } export function buildArenaShareSnapshot(input: ArenaExportInput): ArenaShareSnapshot { const { session, responses, votes, models } = input; const winner = computeWinner(responses, votes); return { version: 1, prompt: session.prompt, systemPrompt: session.systemPrompt, blind: isBlind(session.settings), attachmentCount: attachmentCount(session.settings), parameters: generationParameters(session.settings), createdAt: session.createdAt, models: session.modelKeys.map((key) => ({ key, provider: models[key]?.provider ?? key.split("/")[0], displayName: modelName(key, models) })), responses: responses.map((r) => ({ id: r.id, modelKey: r.modelKey, provider: r.provider, displayName: modelName(r.modelKey, models), status: r.status, content: r.content, reasoning: r.reasoning, error: r.error, ttftMs: r.ttftMs, latencyMs: r.latencyMs, costUsd: r.costUsd, usage: r.usage ? { inputTokens: r.usage.inputTokens, outputTokens: r.usage.outputTokens, reasoningTokens: (r.usage as { reasoningTokens?: number }).reasoningTokens } : null, criteriaWon: votes.filter((v) => v.responseId === r.id).map((v) => v.criterion), })), votes: votes.map((v) => ({ criterion: v.criterion, label: criterionById(v.criterion).label, modelKey: v.modelKey, responseId: v.responseId })), winner: winner ? { modelKey: winner.modelKey, responseId: winner.responseId, criteriaWon: winner.criteriaWon, tieBreak: winner.tieBreak, deltas: winner.deltas } : null, }; }