SPB Git

Repository Public

Arena.swift 16.4 KB 319 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatBenchmarks/Sources/Arena.swift3// Description: Model Arena: runs the same prompt on several models (sequentially or concurrently, memory-aware), streams every answer, supports blind evaluation, votes and a leaderboard.4import Foundation5import LocalChatCore67public enum ArenaError: LocalizedError, Sendable { case runNotFound, unknownContestant, noModels8    public var errorDescription: String? {9        switch self { case .runNotFound: "Arena run not found."; case .unknownContestant: "That answer is not part of this run."; case .noModels: "Pick at least one model." }10    }11}1213public enum ArenaExecutionMode: String, Codable, Sendable, CaseIterable { case auto, sequential, concurrent }1415public struct ArenaConfiguration: Sendable {16    public var prompt: String17    public var systemPrompt: String18    public var settings: GenerationSettings     // set `seed` for repeatable runs19    public var modelIDs: [String]20    public var repetitions: Int21    public var mode: ArenaExecutionMode22    public var blind: Bool23    public var shuffleSeed: UInt64?             // blind label assignment; random (and recorded) when nil24    public var images: [ImageInput]25    public init(prompt: String, systemPrompt: String = "", settings: GenerationSettings = .default, modelIDs: [String], repetitions: Int = 1,26                mode: ArenaExecutionMode = .auto, blind: Bool = false, shuffleSeed: UInt64? = nil, images: [ImageInput] = []) {27        self.prompt = prompt; self.systemPrompt = systemPrompt; self.settings = settings; self.modelIDs = modelIDs; self.repetitions = max(1, repetitions)28        self.mode = mode; self.blind = blind; self.shuffleSeed = shuffleSeed; self.images = images29    }30}3132public struct ArenaContestant: Codable, Sendable, Hashable, Identifiable {33    public var id: UUID34    public var modelID: String35    public var modelName: String36    public var backend: BackendKind37    public var quantization: String?38    public var label: String          // "Model A" in blind mode, the model name otherwise39    public var repetition: Int        // 1-based40    /// What the UI may show: the anonymous label until the run is revealed.41    public func displayName(revealed: Bool) -> String { revealed ? modelName : label }42}4344public struct ArenaEntry: Codable, Sendable, Hashable, Identifiable {45    public var contestant: ArenaContestant46    public var output: String47    public var reasoning: String48    public var metrics: GenerationMetrics?49    public var errorText: String?50    public var id: UUID { contestant.id }51}5253public enum ArenaVerdict: Codable, Sendable, Hashable { case winner(UUID), tie, bothBad }5455public struct ArenaVote: Codable, Sendable, Hashable {56    public var verdict: ArenaVerdict57    public var createdAt: Date58    public init(verdict: ArenaVerdict, createdAt: Date = Date()) { self.verdict = verdict; self.createdAt = createdAt }59}6061public struct ArenaRun: Codable, Sendable, Hashable, Identifiable {62    public var id: UUID63    public var createdAt: Date64    public var prompt: String65    public var systemPrompt: String66    public var settings: GenerationSettings67    public var device: String68    public var blind: Bool69    public var shuffleSeed: UInt64?70    public var ranConcurrently: Bool71    public var wasCancelled: Bool72    public var entries: [ArenaEntry]73    public var votes: [ArenaVote]7475    /// Blind label → model name.76    public func reveal() -> [String: String] {77        Dictionary(entries.map { ($0.contestant.label, $0.contestant.modelName) }, uniquingKeysWith: { a, _ in a })78    }7980    public func exportJSON() throws -> Data { try CoreCoding.prettyEncoder.encode(self) }8182    public func exportMarkdown(revealed: Bool = true) -> String {83        func n(_ v: Double?, _ f: String) -> String { v.map { String(format: f, $0) } ?? "—" }84        var out = "# Arena run\n\n\(createdAt.formatted(date: .abbreviated, time: .shortened)) · device \(device) · \(ranConcurrently ? "concurrent" : "sequential") execution"85        out += blind ? " · blind\n\n" : "\n\n"86        if wasCancelled { out += "*This run was cancelled before every model finished.*\n\n" }87        if !systemPrompt.isEmpty { out += "**System prompt**\n\n\(systemPrompt)\n\n" }88        out += "**Prompt**\n\n\(prompt)\n\n"89        out += "Settings: temperature \(settings.temperature), top-p \(settings.topP), top-k \(settings.topK), max tokens \(settings.maxTokens), seed \(settings.seed.map(String.init) ?? "random")\n\n"90        out += "| Model | Run | TTFT (s) | Tokens/s | Prompt tokens/s | Output tokens | Total (s) | Peak memory |\n|---|---:|---:|---:|---:|---:|---:|---:|\n"91        for e in entries {92            let m = e.metrics93            out += "| \(e.contestant.displayName(revealed: revealed)) | \(e.contestant.repetition) | \(n(m?.timeToFirstToken, "%.2f")) | \(n(m?.tokensPerSecond, "%.1f")) | \(n(m?.promptTokensPerSecond, "%.0f")) | \(m?.completionTokens.map(String.init) ?? "—") | \(n(m?.totalTime, "%.2f")) | \(m?.peakMemoryBytes.map(ByteFormat.memory) ?? "—") |\n"94        }95        out += "\n"96        for e in entries {97            out += "## \(e.contestant.displayName(revealed: revealed))\(e.contestant.repetition > 1 ? " — run \(e.contestant.repetition)" : "")\n\n"98            if let err = e.errorText { out += "*Failed: \(err)*\n\n" }99            if !e.reasoning.isEmpty { out += "<details>\n<summary>Reasoning</summary>\n\n\(e.reasoning)\n\n</details>\n\n" }100            out += e.output + "\n\n"101        }102        if !votes.isEmpty {103            out += "## Votes\n\n"104            for v in votes {105                switch v.verdict {106                case .winner(let id): out += "- Winner: \(entries.first { $0.contestant.id == id }?.contestant.displayName(revealed: revealed) ?? "?")\n"107                case .tie: out += "- Tie\n"108                case .bothBad: out += "- All answers were bad\n"109                }110            }111        }112        return out113    }114}115116public enum ArenaEvent: Sendable {117    case planned(contestants: [ArenaContestant], concurrent: Bool)118    case started(UUID)119    case text(UUID, String)120    case reasoning(UUID, String)121    case finished(UUID, GenerationMetrics)122    case failed(UUID, String)123    case completed(ArenaRun)          // always last; the run is already persisted when a store was given124}125126struct SplitMix64: RandomNumberGenerator {127    var state: UInt64128    mutating func next() -> UInt64 {129        state &+= 0x9E3779B97F4A7C15130        var z = state131        z = (z ^ (z >> 30)) &* 0xBF58476D1CE4E5B9132        z = (z ^ (z >> 27)) &* 0x94D049BB133111EB133        return z ^ (z >> 31)134    }135}136137public final class ArenaRunner: Sendable {138    let service: any InferenceService139    let store: MetricsStore?140    let device: String141    let canRunConcurrently: @Sendable ([ModelDescriptor]) -> Bool142143    /// - Parameter canRunConcurrently: receives the distinct ON-DEVICE models of a run and says whether they fit in memory together. Remote models never count.144    public init(service: any InferenceService, store: MetricsStore? = nil, device: String,145                canRunConcurrently: @escaping @Sendable ([ModelDescriptor]) -> Bool = { $0.count <= 1 }) {146        self.service = service; self.store = store; self.device = device; self.canRunConcurrently = canRunConcurrently147    }148149    static func labels(count: Int, seed: UInt64) -> [String] {150        var names = (0..<count).map { i -> String in151            let letter = String(UnicodeScalar(UInt8(65 + i % 26)))152            return "Model " + (i < 26 ? letter : letter + "\(i / 26 + 1)")153        }154        var rng = SplitMix64(state: seed)155        names.shuffle(using: &rng)156        return names157    }158159    /// Runs the arena. Terminating the stream (or cancelling the consuming task) cancels the generations; what finished is still persisted.160    public func run(_ configuration: ArenaConfiguration) -> AsyncStream<ArenaEvent> {161        AsyncStream { continuation in162            let task = Task { await self.execute(configuration, continuation) }163            continuation.onTermination = { _ in task.cancel() }164        }165    }166167    private func execute(_ config: ArenaConfiguration, _ continuation: AsyncStream<ArenaEvent>.Continuation) async {168        var seen = Set<String>()169        let modelIDs = config.modelIDs.filter { seen.insert($0).inserted }170        let known = Dictionary(await service.listModels().map { ($0.id, $0) }, uniquingKeysWith: { a, _ in a })171        let seed = config.blind ? (config.shuffleSeed ?? UInt64.random(in: 1...UInt64.max)) : config.shuffleSeed172        let blindLabels = config.blind ? Self.labels(count: modelIDs.count, seed: seed ?? 0) : []173174        var rounds: [[ArenaContestant]] = []175        for rep in 1...config.repetitions {176            rounds.append(modelIDs.enumerated().map { i, id in177                let d = known[id]178                return ArenaContestant(id: UUID(), modelID: id, modelName: d?.name ?? id, backend: d?.backend ?? .mlx, quantization: d?.quantization,179                                       label: config.blind ? blindLabels[i] : (d?.name ?? id), repetition: rep)180            })181        }182        // blind mode: present answers in label order so position does not leak the selection order183        if config.blind { rounds = rounds.map { $0.sorted { $0.label < $1.label } } }184185        let local = modelIDs.compactMap { known[$0] }.filter { $0.backend != .remote }186        let unknownLocal = modelIDs.contains { known[$0] == nil }187        let concurrent: Bool = switch config.mode {188        case .sequential: false189        case .concurrent: true190        case .auto: modelIDs.count > 1 && !unknownLocal && (local.isEmpty || canRunConcurrently(local))191        }192        let all = rounds.flatMap { $0 }193        continuation.yield(.planned(contestants: all, concurrent: concurrent))194195        let collector = EntryCollector(all)196        var messages: [PromptMessage] = []197        if !config.systemPrompt.isEmpty { messages.append(.system(config.systemPrompt)) }198        messages.append(.user(config.prompt, images: config.images))199        let request = GenerationRequest(messages: messages, settings: config.settings)200201        for round in rounds {202            if Task.isCancelled { break }203            if concurrent {204                await withTaskGroup(of: Void.self) { group in205                    for c in round { group.addTask { await self.runOne(c, request, collector, continuation) } }206                }207            } else {208                for c in round {209                    if Task.isCancelled { break }210                    await runOne(c, request, collector, continuation)211                }212            }213        }214215        let run = ArenaRun(id: UUID(), createdAt: Date(), prompt: config.prompt, systemPrompt: config.systemPrompt, settings: config.settings, device: device,216                           blind: config.blind, shuffleSeed: seed, ranConcurrently: concurrent, wasCancelled: Task.isCancelled,217                           entries: await collector.entries(), votes: [])218        if let store {219            try? store.saveArenaRun(run)220            for e in run.entries {221                guard let m = e.metrics, m.stopReason != .cancelled else { continue }222                try? store.insert(BenchmarkRecord(modelID: e.contestant.modelID, modelName: e.contestant.modelName, backend: e.contestant.backend,223                                                  quantization: e.contestant.quantization, device: device, kind: "arena",224                                                  promptLabel: String(config.prompt.prefix(60)), metrics: m, contextLength: config.settings.contextLength))225            }226        }227        continuation.yield(.completed(run))228        continuation.finish()229    }230231    private func runOne(_ c: ArenaContestant, _ request: GenerationRequest, _ collector: EntryCollector, _ continuation: AsyncStream<ArenaEvent>.Continuation) async {232        continuation.yield(.started(c.id))233        do {234            var finished = false235            for try await event in service.generate(modelID: c.modelID, request: request) {236                switch event {237                case .text(let t): await collector.append(c.id, text: t); continuation.yield(.text(c.id, t))238                case .reasoning(let t): await collector.append(c.id, reasoning: t); continuation.yield(.reasoning(c.id, t))239                case .finished(let m): finished = true; await collector.finish(c.id, metrics: m); continuation.yield(.finished(c.id, m))240                case .toolCall: break241                }242                if Task.isCancelled { break }243            }244            if !finished {245                let message = Task.isCancelled ? "Cancelled." : "The model stopped without reporting metrics."246                await collector.fail(c.id, message); continuation.yield(.failed(c.id, message))247            }248        } catch {249            let message = Task.isCancelled ? "Cancelled." : ((error as? LocalizedError)?.errorDescription ?? String(describing: error))250            await collector.fail(c.id, message); continuation.yield(.failed(c.id, message))251        }252    }253}254255actor EntryCollector {256    private var order: [UUID]257    private var map: [UUID: ArenaEntry]258    init(_ contestants: [ArenaContestant]) {259        order = contestants.map(\.id)260        map = Dictionary(uniqueKeysWithValues: contestants.map { ($0.id, ArenaEntry(contestant: $0, output: "", reasoning: "", metrics: nil, errorText: nil)) })261    }262    func append(_ id: UUID, text: String) { map[id]?.output += text }263    func append(_ id: UUID, reasoning: String) { map[id]?.reasoning += reasoning }264    func finish(_ id: UUID, metrics: GenerationMetrics) { map[id]?.metrics = metrics }265    func fail(_ id: UUID, _ message: String) { map[id]?.errorText = message }266    func entries() -> [ArenaEntry] { order.compactMap { map[$0] } }267}268269public struct LeaderboardRow: Codable, Sendable, Hashable, Identifiable {270    public var modelID: String271    public var modelName: String272    public var rating: Double         // Elo273    public var wins: Int274    public var losses: Int275    public var ties: Int276    public var comparisons: Int277    public var winRate: Double? { comparisons > 0 ? (Double(wins) + 0.5 * Double(ties)) / Double(comparisons) : nil }278    public var id: String { modelID }279}280281/// Method: Elo ratings (initial 1000, K = 24) over the pairwise outcomes implied by each vote, replayed in chronological order.282/// A "winner" vote beats every other model of that run; "tie" is a draw between every pair; "all bad" changes nothing.283/// `winRate` is the plain share of pairwise comparisons won (ties count half). With few votes both numbers are indicative only.284public enum ArenaLeaderboard {285    public static let initialRating = 1000.0286    public static let k = 24.0287288    public static func compute(_ runsOldestFirst: some Sequence<ArenaRun>) -> [LeaderboardRow] {289        var rows: [String: LeaderboardRow] = [:]290        func row(_ c: ArenaContestant) -> LeaderboardRow {291            rows[c.modelID] ?? LeaderboardRow(modelID: c.modelID, modelName: c.modelName, rating: initialRating, wins: 0, losses: 0, ties: 0, comparisons: 0)292        }293        func play(_ a: ArenaContestant, _ b: ArenaContestant, scoreA: Double) {294            guard a.modelID != b.modelID else { return }295            var ra = row(a), rb = row(b)296            let expectedA = 1 / (1 + pow(10, (rb.rating - ra.rating) / 400))297            ra.rating += k * (scoreA - expectedA); rb.rating += k * ((1 - scoreA) - (1 - expectedA))298            ra.comparisons += 1; rb.comparisons += 1299            if scoreA == 1 { ra.wins += 1; rb.losses += 1 } else if scoreA == 0 { ra.losses += 1; rb.wins += 1 } else { ra.ties += 1; rb.ties += 1 }300            rows[a.modelID] = ra; rows[b.modelID] = rb301        }302        for run in runsOldestFirst {303            var seen = Set<String>()304            let models = run.entries.map(\.contestant).filter { seen.insert($0.modelID).inserted }   // one seat per model305            for vote in run.votes.sorted(by: { $0.createdAt < $1.createdAt }) {306                switch vote.verdict {307                case .bothBad: continue308                case .tie:309                    for i in models.indices { for j in models.indices where j > i { play(models[i], models[j], scoreA: 0.5) } }310                case .winner(let id):311                    guard let w = run.entries.first(where: { $0.contestant.id == id })?.contestant else { continue }312                    for other in models where other.modelID != w.modelID { play(w, other, scoreA: 1) }313                }314            }315        }316        return rows.values.sorted { ($0.rating, $1.modelName) > ($1.rating, $0.modelName) }317    }318}319
Files main
Menu

Theme