SPB Git

Repository Public

BenchmarksTests.swift 16.5 KB 243 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Tests/LocalChatDataTests/BenchmarksTests.swift3// Description: Metrics store aggregates, arena execution (sequential / concurrent / blind / cancel), leaderboard and benchmark runner, against a scripted inference service.4import Foundation5import LocalChatBenchmarks6import LocalChatCore7import Testing89/// Scripted service: streams fixed words with a delay and records how many generations overlap.10final class FakeBenchService: InferenceService, @unchecked Sendable {11    private let lock = NSLock()12    private var active = 013    private(set) var maxConcurrent = 014    private(set) var requests: [(String, GenerationRequest)] = []15    var models: [ModelDescriptor]16    var delay: Duration17    var failing: Set<String> = []1819    init(models: [ModelDescriptor], delay: Duration = .milliseconds(15)) { self.models = models; self.delay = delay }2021    func listModels() async -> [ModelDescriptor] { models }22    func embed(modelID: String, texts: [String]) async throws -> [[Float]] { [] }23    func countTokens(modelID: String, text: String) async -> Int? { nil }2425    func generate(modelID: String, request: GenerationRequest) -> AsyncThrowingStream<GenerationEvent, Error> {26        AsyncThrowingStream { continuation in27            let task = Task {28                self.lock.withLock { self.active += 1; self.maxConcurrent = max(self.maxConcurrent, self.active); self.requests.append((modelID, request)) }29                defer { self.lock.withLock { self.active -= 1 } }30                if self.failing.contains(modelID) { continuation.finish(throwing: InferenceError.loadFailed("boom")); return }31                continuation.yield(.reasoning("hm "))32                for word in ["answer", " from", " \(modelID)"] {33                    do { try await Task.sleep(for: self.delay) } catch { continuation.finish(throwing: InferenceError.cancelled); return }34                    continuation.yield(.text(word))35                }36                continuation.yield(.finished(GenerationMetrics(promptTokens: 20, completionTokens: 30, timeToFirstToken: 0.1, promptTime: 0.2, generationTime: 1.5, totalTime: 1.7)))37                continuation.finish()38            }39            continuation.onTermination = { _ in task.cancel() }40        }41    }42}4344@Suite struct BenchmarksTests {45    static let modelA = ModelDescriptor(id: "mlx:a", name: "Alpha 4B", backend: .mlx, format: .mlx, quantization: "4bit", sizeBytes: 2_000_000_000, localPath: "a")46    static let modelB = ModelDescriptor(id: "mlx:b", name: "Beta 8B", backend: .mlx, format: .mlx, quantization: "8bit", sizeBytes: 8_000_000_000, localPath: "b")47    static let remote = ModelDescriptor(id: "remote:n:c", name: "Gamma remote", backend: .remote, format: .remote)4849    func record(_ model: String, tps: Double, ttft: Double? = 0.2, device: String = "Mac16,5", load: Double? = nil, peak: Int64? = nil,50                stop: StopReason = .endOfTurn, at: TimeInterval = 0) -> BenchmarkRecord {51        BenchmarkRecord(modelID: model, modelName: model.uppercased(), backend: .mlx, quantization: "4bit", device: device, kind: "chat",52                        metrics: GenerationMetrics(promptTokens: 110, cachedPromptTokens: 10, completionTokens: Int(tps * 2), timeToFirstToken: ttft,53                                                   promptTime: 0.5, generationTime: 2, peakMemoryBytes: peak, stopReason: stop),54                        loadTime: load, createdAt: Date(timeIntervalSince1970: 1_700_000_000 + at))55    }5657    @Test func percentiles() {58        #expect(Statistics.median([]) == nil)59        #expect(Statistics.median([3, 1, 2]) == 2)60        #expect(Statistics.median([1, 2, 3, 4]) == 2.5)61        #expect(Statistics.percentile([10, 20, 30, 40, 50], 0.1) == 14)62        #expect(Statistics.percentile([10, 20, 30, 40, 50], 0.9) == 46)63    }6465    @Test func aggregatesWithKnownNumbers() throws {66        let store = try MetricsStore.inMemory()67        for (i, tps) in [10.0, 20, 30, 40, 50].enumerated() { try store.insert(record("m1", tps: tps, ttft: 0.1 * Double(i + 1), load: i == 0 ? 3 : nil, peak: Int64(1000 + i), at: Double(i))) }68        try store.insert(record("m1", tps: 999, stop: .cancelled, at: 9))            // cancelled runs do not skew throughput69        try store.insert(record("m2", tps: 80, at: 1))70        try store.insert(record("m1", tps: 5, device: "iPhone18,2", at: 2))7172        let a = try #require(try store.aggregate(modelID: "m1", device: "Mac16,5"))73        #expect(a.runCount == 6 && a.medianTokensPerSecond == 30 && a.p10TokensPerSecond == 14 && a.p90TokensPerSecond == 46)74        #expect(abs((a.medianTimeToFirstToken ?? 0) - 0.3) < 1e-9)75        #expect(a.medianPromptTokensPerSecond == 200)                                // (110 - 10) / 0.576        #expect(a.medianLoadTime == 3 && a.peakMemoryBytes == 1004)77        #expect(try store.measuredSpeed(device: "Mac16,5") == ["m1": 30, "m2": 80])78        #expect(try store.measuredSpeed(device: "iPhone18,2") == ["m1": 5])79        #expect(try store.devices() == ["Mac16,5", "iPhone18,2"])80        #expect(try store.history(modelID: "m1", device: "Mac16,5").map(\.metrics.completionTokens) == [20, 40, 60, 80, 100, 1998])81        #expect(try store.records(modelID: "m1", device: "Mac16,5", limit: 2).count == 2)8283        let report = try store.comparisonReport(device: "Mac16,5")84        #expect(report.rows.map(\.modelID) == ["m2", "m1"])85        #expect(report.markdown().contains("| M1 | MLX | 4bit | 6 | 30.0 | 14.0–46.0 |"))86        #expect(report.csv().split(separator: "\n").count == 3)87        #expect(try JSONSerialization.jsonObject(with: report.json()) is [String: Any])88        #expect(try store.exportRecordsCSV(modelID: "m2").split(separator: "\n").count == 2)89        #expect(try JSONDecoder.iso.decode([BenchmarkRecord].self, from: store.exportRecordsJSON()).count == 8)9091        // nil stays nil: a record without timings yields no throughput92        try store.insert(BenchmarkRecord(modelID: "m3", modelName: "M3", backend: .remote, device: "Mac16,5", kind: "chat", metrics: GenerationMetrics()))93        let empty = try #require(try store.aggregate(modelID: "m3", device: "Mac16,5"))94        #expect(empty.medianTokensPerSecond == nil && empty.medianTimeToFirstToken == nil && empty.peakMemoryBytes == nil && empty.runCount == 1)9596        #expect(try store.prune(olderThan: Date(timeIntervalSince1970: 1_700_000_000 + 1.5)) == 3)97        try store.deleteRecords(modelID: "m1")98        #expect(try store.records().map(\.modelID) == ["m3"])99    }100101    func collect(_ stream: AsyncStream<ArenaEvent>) async -> (events: [ArenaEvent], run: ArenaRun?) {102        var events: [ArenaEvent] = []; var run: ArenaRun?103        for await e in stream { events.append(e); if case .completed(let r) = e { run = r } }104        return (events, run)105    }106107    @Test func arenaSequentialWhenMemoryDoesNotAllow() async throws {108        let service = FakeBenchService(models: [Self.modelA, Self.modelB])109        let store = try MetricsStore.inMemory()110        let runner = ArenaRunner(service: service, store: store, device: "Mac16,5", canRunConcurrently: { _ in false })111        var settings = GenerationSettings.default; settings.seed = 7112        let (events, result) = await collect(runner.run(ArenaConfiguration(prompt: "Hi", systemPrompt: "Be brief", settings: settings,113                                                                             modelIDs: ["mlx:a", "mlx:b", "mlx:a"], repetitions: 2)))114        let run = try #require(result)115        #expect(service.maxConcurrent == 1 && !run.ranConcurrently && !run.wasCancelled)116        #expect(run.entries.count == 4)                                              // duplicates removed, 2 models × 2 repetitions117        #expect(run.entries.map(\.contestant.repetition) == [1, 1, 2, 2])118        #expect(run.entries.allSatisfy { $0.output == "answer from \($0.contestant.modelID)" && $0.reasoning == "hm " && $0.metrics?.tokensPerSecond == 20 })119        #expect(run.entries[0].contestant.label == "Alpha 4B" && run.entries[0].contestant.quantization == "4bit")120        #expect(service.requests.allSatisfy { $0.1.settings.seed == 7 && $0.1.messages.first?.role == .system })121        if case .planned(let contestants, let concurrent) = events.first { #expect(contestants.count == 4 && !concurrent) } else { Issue.record("planned must come first") }122        #expect(try store.arenaRun(run.id) == run)123        #expect(try store.records(kind: "arena").count == 4)124        #expect(run.exportMarkdown().contains("## Beta 8B — run 2") && run.exportMarkdown().contains("seed 7"))125        #expect(try JSONDecoder.iso.decode(ArenaRun.self, from: run.exportJSON()).id == run.id)126        try store.deleteArenaRun(run.id)127        #expect(try store.arenaRuns().isEmpty)128    }129130    @Test func arenaConcurrentWhenAllowedAndRemoteNeverBlocks() async throws {131        let service = FakeBenchService(models: [Self.modelA, Self.modelB, Self.remote], delay: .milliseconds(40))132        let asked = LockedBox<[[String]]>([])133        let runner = ArenaRunner(service: service, device: "Mac16,5", canRunConcurrently: { models in asked.mutate { $0.append(models.map(\.id)) }; return true })134        let (_, result) = await collect(runner.run(ArenaConfiguration(prompt: "Hi", modelIDs: ["mlx:a", "mlx:b", "remote:n:c"])))135        #expect(try #require(result).ranConcurrently && service.maxConcurrent == 3)136        #expect(asked.value == [["mlx:a", "mlx:b"]])                                 // only on-device models are weighed137138        // one local + one remote, memory closure refusing pairs: a single local model still passes the default rule139        let service2 = FakeBenchService(models: [Self.modelA, Self.remote], delay: .milliseconds(40))140        let (_, r2) = await collect(ArenaRunner(service: service2, device: "d").run(ArenaConfiguration(prompt: "Hi", modelIDs: ["mlx:a", "remote:n:c"])))141        #expect(try #require(r2).ranConcurrently && service2.maxConcurrent == 2)142143        // forced modes override the decision; failures are reported per contestant144        let service3 = FakeBenchService(models: [Self.modelA, Self.modelB])145        service3.failing = ["mlx:b"]146        let (events, r3) = await collect(ArenaRunner(service: service3, device: "d", canRunConcurrently: { _ in true })147            .run(ArenaConfiguration(prompt: "Hi", modelIDs: ["mlx:a", "mlx:b"], mode: .sequential)))148        let run3 = try #require(r3)149        #expect(!run3.ranConcurrently && service3.maxConcurrent == 1)150        #expect(run3.entries[1].errorText?.contains("boom") == true && run3.entries[1].metrics == nil && run3.entries[0].metrics != nil)151        #expect(events.contains { if case .failed = $0 { true } else { false } })152    }153154    @Test func arenaBlindLabelsVotesAndLeaderboard() async throws {155        let service = FakeBenchService(models: [Self.modelA, Self.modelB, Self.remote], delay: .milliseconds(2))156        let store = try MetricsStore.inMemory()157        let runner = ArenaRunner(service: service, store: store, device: "d")158        let config = ArenaConfiguration(prompt: "Hi", modelIDs: ["mlx:a", "mlx:b", "remote:n:c"], mode: .sequential, blind: true, shuffleSeed: 42)159        let run1 = try #require(await collect(runner.run(config)).run)160        let run2 = try #require(await collect(runner.run(config)).run)161        #expect(run1.entries.map(\.contestant.label) == ["Model A", "Model B", "Model C"])       // presented in label order162        #expect(run1.reveal() == run2.reveal() && run1.shuffleSeed == 42)                        // stable for a recorded seed163        #expect(Set(run1.reveal().values) == ["Alpha 4B", "Beta 8B", "Gamma remote"])164        #expect(run1.entries[0].contestant.displayName(revealed: false) == "Model A")165        #expect(!run1.exportMarkdown(revealed: false).contains("Alpha 4B") && run1.exportMarkdown(revealed: true).contains("Alpha 4B"))166        let random = try #require(await collect(runner.run(ArenaConfiguration(prompt: "Hi", modelIDs: ["mlx:a", "mlx:b"], blind: true))).run)167        #expect(random.shuffleSeed != nil)168169        let alpha = try #require(run1.entries.first { $0.contestant.modelID == "mlx:a" }).contestant170        let voted = try store.recordVote(.winner(alpha.id), runID: run1.id)171        #expect(voted.votes.count == 1)172        try store.recordVote(.tie, runID: run2.id)173        try store.recordVote(.bothBad, runID: run2.id)174        #expect(throws: ArenaError.self) { try store.recordVote(.winner(UUID()), runID: run1.id) }175        #expect(throws: ArenaError.self) { try store.recordVote(.tie, runID: UUID()) }176177        let board = try store.leaderboard()178        let top = try #require(board.first)179        #expect(top.modelID == "mlx:a" && top.wins == 2 && top.ties == 2 && top.losses == 0 && top.comparisons == 4 && top.rating > 1000)180        #expect(top.winRate == 0.75)181        #expect(board.count == 3 && abs(board.map(\.rating).reduce(0, +) - 3000) < 1e-6)        // Elo is zero-sum182    }183184    @Test func arenaCancelKeepsWhatFinished() async throws {185        let service = FakeBenchService(models: [Self.modelA, Self.modelB], delay: .milliseconds(120))186        let store = try MetricsStore.inMemory()187        let runner = ArenaRunner(service: service, store: store, device: "d")188        let consumer = Task {189            var texts = 0190            for await e in runner.run(ArenaConfiguration(prompt: "Hi", modelIDs: ["mlx:a", "mlx:b"], mode: .sequential)) {191                if case .text = e { texts += 1; if texts == 1 { break } }               // stop consuming after the first token192            }193        }194        await consumer.value195        var saved: [ArenaRun] = []196        for _ in 0..<100 { saved = try store.arenaRuns(); if !saved.isEmpty { break }; try await Task.sleep(for: .milliseconds(20)) }197        let run = try #require(saved.first)198        #expect(run.wasCancelled && run.entries.count == 2)199        #expect(run.entries.allSatisfy { $0.metrics == nil })200        #expect(run.entries[0].output.hasPrefix("answer") && run.entries[1].output.isEmpty)201        #expect(try store.records(kind: "arena").isEmpty)202        #expect(service.requests.count == 1)                                          // the second model never started203    }204205    @Test func benchmarkRunnerRecordsEveryPrompt() async throws {206        let suite = BenchmarkSuite.standard207        #expect(suite.version == 1 && suite.prompts.map(\.id) == ["short-answer", "long-generation", "long-prompt", "code"])208        let long = try #require(suite.prompts.first { $0.id == "long-prompt" })209        #expect((1500...2600).contains(TokenEstimator.estimate(long.prompt)))        // ≈ 2k tokens by the character estimate210        #expect(Set(suite.prompts.map(\.seed)).count == suite.prompts.count)211212        let service = FakeBenchService(models: [Self.modelA], delay: .milliseconds(1))213        let store = try MetricsStore.inMemory()214        var progress: [BenchmarkProgress] = []215        for await p in BenchmarkRunner(service: service, store: store, device: "Mac16,5").run(modelID: "mlx:a", loadTime: 2.5) { progress.append(p) }216        guard case .completed(let summary) = progress.last else { Issue.record("completed must be last"); return }217        #expect(summary.records.count == 4 && summary.failures.isEmpty && !summary.wasCancelled)218        #expect(summary.medianTokensPerSecond == 20 && summary.medianTimeToFirstToken == 0.1 && summary.medianPromptTokensPerSecond == 100)219        #expect(summary.peakMemoryBytes == nil)                                       // the backend measured none: stays nil220        #expect(summary.records.map(\.promptLabel) == suite.prompts.map { "standard-v1/\($0.id)" })221        #expect(summary.records.map(\.loadTime) == [2.5, nil, nil, nil] && summary.records.allSatisfy { $0.kind == "benchmark" && $0.modelName == "Alpha 4B" })222        #expect(zip(service.requests, suite.prompts).allSatisfy { $0.0.1.settings.seed == $0.1.seed && $0.0.1.settings.maxTokens == $0.1.maxTokens })223        #expect(try store.records(kind: "benchmark").count == 4)224        #expect(try store.aggregate(modelID: "mlx:a", device: "Mac16,5")?.medianLoadTime == 2.5)225226        service.failing = ["mlx:a"]227        var failed: BenchmarkSummary?228        for await p in BenchmarkRunner(service: service, device: "d").run(modelID: "mlx:a") { if case .completed(let s) = p { failed = s } }229        #expect(failed?.records.isEmpty == true && failed?.failures.count == 4 && failed?.medianTokensPerSecond == nil)230    }231}232233final class LockedBox<T>: @unchecked Sendable {234    private let lock = NSLock(); private var stored: T235    init(_ value: T) { stored = value }236    var value: T { lock.withLock { stored } }237    func mutate(_ body: (inout T) -> Void) { lock.withLock { body(&stored) } }238}239240extension JSONDecoder {241    static var iso: JSONDecoder { let d = JSONDecoder(); d.dateDecodingStrategy = .iso8601; return d }242}243
Files main
Menu

Theme