Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatBenchmarks/Sources/Arena.swift3// Description: Model Arena: runs the same prompt on several models (sequentially or concurrently, memory-aware), streams every answer, supports blind evaluation, votes and a leaderboard.4import Foundation5import LocalChatCore67public enum ArenaError: LocalizedError, Sendable { case runNotFound, unknownContestant, noModels8 public var errorDescription: String? {9 switch self { case .runNotFound: "Arena run not found."; case .unknownContestant: "That answer is not part of this run."; case .noModels: "Pick at least one model." }10 }11}1213public enum ArenaExecutionMode: String, Codable, Sendable, CaseIterable { case auto, sequential, concurrent }1415public struct ArenaConfiguration: Sendable {16 public var prompt: String17 public var systemPrompt: String18 public var settings: GenerationSettings // set `seed` for repeatable runs19 public var modelIDs: [String]20 public var repetitions: Int21 public var mode: ArenaExecutionMode22 public var blind: Bool23 public var shuffleSeed: UInt64? // blind label assignment; random (and recorded) when nil24 public var images: [ImageInput]25 public init(prompt: String, systemPrompt: String = "", settings: GenerationSettings = .default, modelIDs: [String], repetitions: Int = 1,26 mode: ArenaExecutionMode = .auto, blind: Bool = false, shuffleSeed: UInt64? = nil, images: [ImageInput] = []) {27 self.prompt = prompt; self.systemPrompt = systemPrompt; self.settings = settings; self.modelIDs = modelIDs; self.repetitions = max(1, repetitions)28 self.mode = mode; self.blind = blind; self.shuffleSeed = shuffleSeed; self.images = images29 }30}3132public struct ArenaContestant: Codable, Sendable, Hashable, Identifiable {33 public var id: UUID34 public var modelID: String35 public var modelName: String36 public var backend: BackendKind37 public var quantization: String?38 public var label: String // "Model A" in blind mode, the model name otherwise39 public var repetition: Int // 1-based40 /// What the UI may show: the anonymous label until the run is revealed.41 public func displayName(revealed: Bool) -> String { revealed ? modelName : label }42}4344public struct ArenaEntry: Codable, Sendable, Hashable, Identifiable {45 public var contestant: ArenaContestant46 public var output: String47 public var reasoning: String48 public var metrics: GenerationMetrics?49 public var errorText: String?50 public var id: UUID { contestant.id }51}5253public enum ArenaVerdict: Codable, Sendable, Hashable { case winner(UUID), tie, bothBad }5455public struct ArenaVote: Codable, Sendable, Hashable {56 public var verdict: ArenaVerdict57 public var createdAt: Date58 public init(verdict: ArenaVerdict, createdAt: Date = Date()) { self.verdict = verdict; self.createdAt = createdAt }59}6061public struct ArenaRun: Codable, Sendable, Hashable, Identifiable {62 public var id: UUID63 public var createdAt: Date64 public var prompt: String65 public var systemPrompt: String66 public var settings: GenerationSettings67 public var device: String68 public var blind: Bool69 public var shuffleSeed: UInt64?70 public var ranConcurrently: Bool71 public var wasCancelled: Bool72 public var entries: [ArenaEntry]73 public var votes: [ArenaVote]7475 /// Blind label → model name.76 public func reveal() -> [String: String] {77 Dictionary(entries.map { ($0.contestant.label, $0.contestant.modelName) }, uniquingKeysWith: { a, _ in a })78 }7980 public func exportJSON() throws -> Data { try CoreCoding.prettyEncoder.encode(self) }8182 public func exportMarkdown(revealed: Bool = true) -> String {83 func n(_ v: Double?, _ f: String) -> String { v.map { String(format: f, $0) } ?? "—" }84 var out = "# Arena run\n\n\(createdAt.formatted(date: .abbreviated, time: .shortened)) · device \(device) · \(ranConcurrently ? "concurrent" : "sequential") execution"85 out += blind ? " · blind\n\n" : "\n\n"86 if wasCancelled { out += "*This run was cancelled before every model finished.*\n\n" }87 if !systemPrompt.isEmpty { out += "**System prompt**\n\n\(systemPrompt)\n\n" }88 out += "**Prompt**\n\n\(prompt)\n\n"89 out += "Settings: temperature \(settings.temperature), top-p \(settings.topP), top-k \(settings.topK), max tokens \(settings.maxTokens), seed \(settings.seed.map(String.init) ?? "random")\n\n"90 out += "| Model | Run | TTFT (s) | Tokens/s | Prompt tokens/s | Output tokens | Total (s) | Peak memory |\n|---|---:|---:|---:|---:|---:|---:|---:|\n"91 for e in entries {92 let m = e.metrics93 out += "| \(e.contestant.displayName(revealed: revealed)) | \(e.contestant.repetition) | \(n(m?.timeToFirstToken, "%.2f")) | \(n(m?.tokensPerSecond, "%.1f")) | \(n(m?.promptTokensPerSecond, "%.0f")) | \(m?.completionTokens.map(String.init) ?? "—") | \(n(m?.totalTime, "%.2f")) | \(m?.peakMemoryBytes.map(ByteFormat.memory) ?? "—") |\n"94 }95 out += "\n"96 for e in entries {97 out += "## \(e.contestant.displayName(revealed: revealed))\(e.contestant.repetition > 1 ? " — run \(e.contestant.repetition)" : "")\n\n"98 if let err = e.errorText { out += "*Failed: \(err)*\n\n" }99 if !e.reasoning.isEmpty { out += "<details>\n<summary>Reasoning</summary>\n\n\(e.reasoning)\n\n</details>\n\n" }100 out += e.output + "\n\n"101 }102 if !votes.isEmpty {103 out += "## Votes\n\n"104 for v in votes {105 switch v.verdict {106 case .winner(let id): out += "- Winner: \(entries.first { $0.contestant.id == id }?.contestant.displayName(revealed: revealed) ?? "?")\n"107 case .tie: out += "- Tie\n"108 case .bothBad: out += "- All answers were bad\n"109 }110 }111 }112 return out113 }114}115116public enum ArenaEvent: Sendable {117 case planned(contestants: [ArenaContestant], concurrent: Bool)118 case started(UUID)119 case text(UUID, String)120 case reasoning(UUID, String)121 case finished(UUID, GenerationMetrics)122 case failed(UUID, String)123 case completed(ArenaRun) // always last; the run is already persisted when a store was given124}125126struct SplitMix64: RandomNumberGenerator {127 var state: UInt64128 mutating func next() -> UInt64 {129 state &+= 0x9E3779B97F4A7C15130 var z = state131 z = (z ^ (z >> 30)) &* 0xBF58476D1CE4E5B9132 z = (z ^ (z >> 27)) &* 0x94D049BB133111EB133 return z ^ (z >> 31)134 }135}136137public final class ArenaRunner: Sendable {138 let service: any InferenceService139 let store: MetricsStore?140 let device: String141 let canRunConcurrently: @Sendable ([ModelDescriptor]) -> Bool142143 /// - Parameter canRunConcurrently: receives the distinct ON-DEVICE models of a run and says whether they fit in memory together. Remote models never count.144 public init(service: any InferenceService, store: MetricsStore? = nil, device: String,145 canRunConcurrently: @escaping @Sendable ([ModelDescriptor]) -> Bool = { $0.count <= 1 }) {146 self.service = service; self.store = store; self.device = device; self.canRunConcurrently = canRunConcurrently147 }148149 static func labels(count: Int, seed: UInt64) -> [String] {150 var names = (0..<count).map { i -> String in151 let letter = String(UnicodeScalar(UInt8(65 + i % 26)))152 return "Model " + (i < 26 ? letter : letter + "\(i / 26 + 1)")153 }154 var rng = SplitMix64(state: seed)155 names.shuffle(using: &rng)156 return names157 }158159 /// Runs the arena. Terminating the stream (or cancelling the consuming task) cancels the generations; what finished is still persisted.160 public func run(_ configuration: ArenaConfiguration) -> AsyncStream<ArenaEvent> {161 AsyncStream { continuation in162 let task = Task { await self.execute(configuration, continuation) }163 continuation.onTermination = { _ in task.cancel() }164 }165 }166167 private func execute(_ config: ArenaConfiguration, _ continuation: AsyncStream<ArenaEvent>.Continuation) async {168 var seen = Set<String>()169 let modelIDs = config.modelIDs.filter { seen.insert($0).inserted }170 let known = Dictionary(await service.listModels().map { ($0.id, $0) }, uniquingKeysWith: { a, _ in a })171 let seed = config.blind ? (config.shuffleSeed ?? UInt64.random(in: 1...UInt64.max)) : config.shuffleSeed172 let blindLabels = config.blind ? Self.labels(count: modelIDs.count, seed: seed ?? 0) : []173174 var rounds: [[ArenaContestant]] = []175 for rep in 1...config.repetitions {176 rounds.append(modelIDs.enumerated().map { i, id in177 let d = known[id]178 return ArenaContestant(id: UUID(), modelID: id, modelName: d?.name ?? id, backend: d?.backend ?? .mlx, quantization: d?.quantization,179 label: config.blind ? blindLabels[i] : (d?.name ?? id), repetition: rep)180 })181 }182 // blind mode: present answers in label order so position does not leak the selection order183 if config.blind { rounds = rounds.map { $0.sorted { $0.label < $1.label } } }184185 let local = modelIDs.compactMap { known[$0] }.filter { $0.backend != .remote }186 let unknownLocal = modelIDs.contains { known[$0] == nil }187 let concurrent: Bool = switch config.mode {188 case .sequential: false189 case .concurrent: true190 case .auto: modelIDs.count > 1 && !unknownLocal && (local.isEmpty || canRunConcurrently(local))191 }192 let all = rounds.flatMap { $0 }193 continuation.yield(.planned(contestants: all, concurrent: concurrent))194195 let collector = EntryCollector(all)196 var messages: [PromptMessage] = []197 if !config.systemPrompt.isEmpty { messages.append(.system(config.systemPrompt)) }198 messages.append(.user(config.prompt, images: config.images))199 let request = GenerationRequest(messages: messages, settings: config.settings)200201 for round in rounds {202 if Task.isCancelled { break }203 if concurrent {204 await withTaskGroup(of: Void.self) { group in205 for c in round { group.addTask { await self.runOne(c, request, collector, continuation) } }206 }207 } else {208 for c in round {209 if Task.isCancelled { break }210 await runOne(c, request, collector, continuation)211 }212 }213 }214215 let run = ArenaRun(id: UUID(), createdAt: Date(), prompt: config.prompt, systemPrompt: config.systemPrompt, settings: config.settings, device: device,216 blind: config.blind, shuffleSeed: seed, ranConcurrently: concurrent, wasCancelled: Task.isCancelled,217 entries: await collector.entries(), votes: [])218 if let store {219 try? store.saveArenaRun(run)220 for e in run.entries {221 guard let m = e.metrics, m.stopReason != .cancelled else { continue }222 try? store.insert(BenchmarkRecord(modelID: e.contestant.modelID, modelName: e.contestant.modelName, backend: e.contestant.backend,223 quantization: e.contestant.quantization, device: device, kind: "arena",224 promptLabel: String(config.prompt.prefix(60)), metrics: m, contextLength: config.settings.contextLength))225 }226 }227 continuation.yield(.completed(run))228 continuation.finish()229 }230231 private func runOne(_ c: ArenaContestant, _ request: GenerationRequest, _ collector: EntryCollector, _ continuation: AsyncStream<ArenaEvent>.Continuation) async {232 continuation.yield(.started(c.id))233 do {234 var finished = false235 for try await event in service.generate(modelID: c.modelID, request: request) {236 switch event {237 case .text(let t): await collector.append(c.id, text: t); continuation.yield(.text(c.id, t))238 case .reasoning(let t): await collector.append(c.id, reasoning: t); continuation.yield(.reasoning(c.id, t))239 case .finished(let m): finished = true; await collector.finish(c.id, metrics: m); continuation.yield(.finished(c.id, m))240 case .toolCall: break241 }242 if Task.isCancelled { break }243 }244 if !finished {245 let message = Task.isCancelled ? "Cancelled." : "The model stopped without reporting metrics."246 await collector.fail(c.id, message); continuation.yield(.failed(c.id, message))247 }248 } catch {249 let message = Task.isCancelled ? "Cancelled." : ((error as? LocalizedError)?.errorDescription ?? String(describing: error))250 await collector.fail(c.id, message); continuation.yield(.failed(c.id, message))251 }252 }253}254255actor EntryCollector {256 private var order: [UUID]257 private var map: [UUID: ArenaEntry]258 init(_ contestants: [ArenaContestant]) {259 order = contestants.map(\.id)260 map = Dictionary(uniqueKeysWithValues: contestants.map { ($0.id, ArenaEntry(contestant: $0, output: "", reasoning: "", metrics: nil, errorText: nil)) })261 }262 func append(_ id: UUID, text: String) { map[id]?.output += text }263 func append(_ id: UUID, reasoning: String) { map[id]?.reasoning += reasoning }264 func finish(_ id: UUID, metrics: GenerationMetrics) { map[id]?.metrics = metrics }265 func fail(_ id: UUID, _ message: String) { map[id]?.errorText = message }266 func entries() -> [ArenaEntry] { order.compactMap { map[$0] } }267}268269public struct LeaderboardRow: Codable, Sendable, Hashable, Identifiable {270 public var modelID: String271 public var modelName: String272 public var rating: Double // Elo273 public var wins: Int274 public var losses: Int275 public var ties: Int276 public var comparisons: Int277 public var winRate: Double? { comparisons > 0 ? (Double(wins) + 0.5 * Double(ties)) / Double(comparisons) : nil }278 public var id: String { modelID }279}280281/// Method: Elo ratings (initial 1000, K = 24) over the pairwise outcomes implied by each vote, replayed in chronological order.282/// A "winner" vote beats every other model of that run; "tie" is a draw between every pair; "all bad" changes nothing.283/// `winRate` is the plain share of pairwise comparisons won (ties count half). With few votes both numbers are indicative only.284public enum ArenaLeaderboard {285 public static let initialRating = 1000.0286 public static let k = 24.0287288 public static func compute(_ runsOldestFirst: some Sequence<ArenaRun>) -> [LeaderboardRow] {289 var rows: [String: LeaderboardRow] = [:]290 func row(_ c: ArenaContestant) -> LeaderboardRow {291 rows[c.modelID] ?? LeaderboardRow(modelID: c.modelID, modelName: c.modelName, rating: initialRating, wins: 0, losses: 0, ties: 0, comparisons: 0)292 }293 func play(_ a: ArenaContestant, _ b: ArenaContestant, scoreA: Double) {294 guard a.modelID != b.modelID else { return }295 var ra = row(a), rb = row(b)296 let expectedA = 1 / (1 + pow(10, (rb.rating - ra.rating) / 400))297 ra.rating += k * (scoreA - expectedA); rb.rating += k * ((1 - scoreA) - (1 - expectedA))298 ra.comparisons += 1; rb.comparisons += 1299 if scoreA == 1 { ra.wins += 1; rb.losses += 1 } else if scoreA == 0 { ra.losses += 1; rb.wins += 1 } else { ra.ties += 1; rb.ties += 1 }300 rows[a.modelID] = ra; rows[b.modelID] = rb301 }302 for run in runsOldestFirst {303 var seen = Set<String>()304 let models = run.entries.map(\.contestant).filter { seen.insert($0.modelID).inserted } // one seat per model305 for vote in run.votes.sorted(by: { $0.createdAt < $1.createdAt }) {306 switch vote.verdict {307 case .bothBad: continue308 case .tie:309 for i in models.indices { for j in models.indices where j > i { play(models[i], models[j], scoreA: 0.5) } }310 case .winner(let id):311 guard let w = run.entries.first(where: { $0.contestant.id == id })?.contestant else { continue }312 for other in models where other.modelID != w.modelID { play(w, other, scoreA: 1) }313 }314 }315 }316 return rows.values.sorted { ($0.rating, $1.modelName) > ($1.rating, $0.modelName) }317 }318}319