SPB Git

Repository Public

ArenaModel.swift 7.2 KB 150 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatUI/Sources/Features/Arena/ArenaModel.swift3// Description: State of the Model Arena: configuration, live streams per contestant, memory-aware execution decision, votes, past runs and leaderboard.4import Foundation5import Observation6import LocalChatBenchmarks7import LocalChatCore89@MainActor @Observable10final class ArenaModel {11    struct Live: Hashable {12        var text = ""; var reasoning = ""; var metrics: GenerationMetrics?; var error: String?; var started = false13        var isDone: Bool { metrics != nil || error != nil }14    }1516    // configuration17    var modelIDs: [String] = []18    var prompt = ""19    var systemPrompt = ""20    var preset: InferencePreset = .balanced21    var maxTokens = 51222    var temperature = 0.723    var fixedSeed = true24    var seed: UInt64 = 4225    var repetitions = 126    var mode: ArenaExecutionMode = .auto27    var blind = false2829    // run state30    var contestants: [ArenaContestant] = []31    var live: [UUID: Live] = [:]32    var isRunning = false33    var ranConcurrently: Bool?34    var modeReason = ""35    var run: ArenaRun?36    var wasBlindRequested = false37    var errorText: String?3839    // history40    var pastRuns: [ArenaRun] = []41    var leaderboard: [LeaderboardRow] = []4243    @ObservationIgnored private var task: Task<Void, Never>?4445    static let maxModels = 44647    var revealed: Bool { !(run?.blind ?? wasBlindRequested) || !(run?.votes.isEmpty ?? true) }48    var canRun: Bool { !isRunning && modelIDs.count >= 2 && !prompt.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty }4950    func entry(_ c: ArenaContestant) -> Live { live[c.id] ?? Live() }5152    func reload(_ app: AppModel) {53        pastRuns = (try? app.metrics.arenaRuns(limit: 100)) ?? []54        leaderboard = (try? app.metrics.leaderboard()) ?? []55        modelIDs.removeAll { app.model($0) == nil }56    }5758    func add(_ model: ModelDescriptor) {59        guard !modelIDs.contains(model.id), modelIDs.count < Self.maxModels else { return }60        modelIDs.append(model.id)61    }6263    /// Whether the on-device contestants fit in memory together, and the sentence that explains it.64    static func concurrencyDecision(models: [ModelDescriptor], usable: Int64, context: Int) -> (fits: Bool, reason: String) {65        let local = models.filter { $0.backend == .mlx || $0.backend == .gguf }66        if local.count <= 1 { return (true, local.isEmpty ? "No model runs on this device, so the answers can be generated at the same time." : "Only one model runs on this device.") }67        var total: Int64 = 068        for m in local {69            guard let need = m.estimatedMemoryBytes(context: context) else { return (false, "The memory need of \(m.name) is unknown, so the models run one after the other.") }70            total += need71        }72        if total <= usable { return (true, "The on-device models need about \(ByteFormat.memory(total)) together; \(ByteFormat.memory(usable)) is available to models.") }73        return (false, "The on-device models need about \(ByteFormat.memory(total)) together but only \(ByteFormat.memory(usable)) is available to models.")74    }7576    func start(_ app: AppModel) {77        guard canRun else { return }78        app.refreshDevice()79        var settings = preset.settings(maxContext: 32768)80        settings.maxTokens = maxTokens; settings.temperature = temperature; settings.seed = fixedSeed ? seed : nil81        let models = modelIDs.compactMap { app.model($0) }82        let usable = app.device.usableModelMemoryBytes, context = settings.contextLength83        let decision = Self.concurrencyDecision(models: models, usable: usable, context: context)84        let runner = ArenaRunner(service: app.engine, store: app.metrics, device: app.device.machine) { local in85            ArenaModel.fits(local, usable: usable, context: context)86        }87        let configuration = ArenaConfiguration(prompt: prompt.trimmingCharacters(in: .whitespacesAndNewlines), systemPrompt: systemPrompt.trimmingCharacters(in: .whitespacesAndNewlines),88                                               settings: settings, modelIDs: modelIDs, repetitions: repetitions, mode: mode, blind: blind)89        let requestedMode = mode90        contestants = []; live = [:]; run = nil; ranConcurrently = nil; modeReason = ""; errorText = nil; isRunning = true; wasBlindRequested = blind91        task = Task { [weak self] in92            for await event in runner.run(configuration) {93                guard let self else { return }94                switch event {95                case .planned(let all, let concurrent):96                    self.contestants = all; self.ranConcurrently = concurrent97                    for c in all { self.live[c.id] = Live() }98                    switch requestedMode {99                    case .auto: self.modeReason = decision.reason100                    case .sequential: self.modeReason = "You asked for one model at a time."101                    case .concurrent: self.modeReason = "You forced concurrent execution. " + (decision.fits ? "" : "Warning: " + decision.reason)102                    }103                case .started(let id): self.live[id]?.started = true104                case .text(let id, let t): self.live[id]?.text += t105                case .reasoning(let id, let t): self.live[id]?.reasoning += t106                case .finished(let id, let m): self.live[id]?.metrics = m107                case .failed(let id, let message): self.live[id]?.error = message108                case .completed(let run):109                    self.run = run; self.isRunning = false110                    self.reload(app); app.refreshMeasuredSpeed()111                    Haptics.play(.success)112                }113            }114            self?.isRunning = false115        }116    }117118    nonisolated static func fits(_ local: [ModelDescriptor], usable: Int64, context: Int) -> Bool {119        concurrencyDecisionNonisolated(local, usable: usable, context: context)120    }121    nonisolated private static func concurrencyDecisionNonisolated(_ local: [ModelDescriptor], usable: Int64, context: Int) -> Bool {122        if local.count <= 1 { return true }123        var total: Int64 = 0124        for m in local { guard let need = m.estimatedMemoryBytes(context: context) else { return false }; total += need }125        return total <= usable126    }127128    func cancel() { task?.cancel() }129130    func vote(_ verdict: ArenaVerdict, app: AppModel) {131        guard let current = run else { return }132        do { run = try app.metrics.recordVote(verdict, runID: current.id); reload(app); Haptics.play(.selection) }133        catch { errorText = error.localizedDescription }134    }135136    func open(_ past: ArenaRun) {137        guard !isRunning else { return }138        run = past; contestants = past.entries.map(\.contestant); wasBlindRequested = past.blind139        ranConcurrently = past.ranConcurrently; modeReason = "Recorded run."140        live = Dictionary(uniqueKeysWithValues: past.entries.map { ($0.contestant.id, Live(text: $0.output, reasoning: $0.reasoning, metrics: $0.metrics, error: $0.errorText, started: true)) })141        prompt = past.prompt; systemPrompt = past.systemPrompt142    }143144    func delete(_ past: ArenaRun, app: AppModel) {145        try? app.metrics.deleteArenaRun(past.id)146        if run?.id == past.id { run = nil; contestants = []; live = [:] }147        reload(app)148    }149}150
Files main
Menu

Theme