SPB Git

Repository Public

BenchmarkSuite.swift 10.3 KB 137 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatBenchmarks/Sources/BenchmarkSuite.swift3// Description: A fixed, versioned benchmark suite and its runner. Every figure comes from the backend's own measurements.4import Foundation5import LocalChatCore67public struct BenchmarkPrompt: Codable, Sendable, Hashable, Identifiable {8    public var id: String9    public var label: String10    public var purpose: String11    public var systemPrompt: String12    public var prompt: String13    public var maxTokens: Int14    public var seed: UInt6415    public var temperature: Double16}1718public struct BenchmarkSuite: Codable, Sendable, Hashable, Identifiable {19    public var id: String20    public var version: Int21    public var name: String22    public var prompts: [BenchmarkPrompt]2324    /// Changing any prompt, seed or limit requires bumping `version`: results are only comparable within one version.25    public static let standard = BenchmarkSuite(id: "standard", version: 1, name: "Standard suite", prompts: [26        BenchmarkPrompt(id: "short-answer", label: "Short answer", purpose: "Time to first token and latency of a brief reply.",27                        systemPrompt: "You are a concise assistant.", prompt: "In one sentence, what is the capital of Canada and on which river does it lie?",28                        maxTokens: 64, seed: 11, temperature: 0),29        BenchmarkPrompt(id: "long-generation", label: "Long generation", purpose: "Sustained generation throughput.",30                        systemPrompt: "You are a clear technical writer.",31                        prompt: "Write a detailed, well-structured explanation (about 500 words) of how a hash table works, covering hashing, collisions, load factor and resizing.",32                        maxTokens: 700, seed: 23, temperature: 0.7),33        BenchmarkPrompt(id: "long-prompt", label: "Long prompt ingestion", purpose: "Prompt processing throughput on a prompt of roughly two thousand tokens.",34                        systemPrompt: "You are a careful reader.", prompt: longPassage + "\n\nList the three most important points of the text above as short bullet points.",35                        maxTokens: 160, seed: 37, temperature: 0),36        BenchmarkPrompt(id: "code", label: "Code", purpose: "Code generation throughput.",37                        systemPrompt: "You are a senior software engineer. Answer with code first.",38                        prompt: "Write a Swift function that merges two sorted arrays of integers into one sorted array without using sort(), then give two test cases.",39                        maxTokens: 400, seed: 41, temperature: 0.2),40    ])4142    /// Deterministic passage (≈ 8 000 characters, about two thousand tokens with common tokenizers; the exact count is measured, never assumed).43    static let longPassage: String = {44        let paragraphs = [45            "Unified memory changes how a language model is deployed on a personal computer. Instead of copying weights from system memory to a separate graphics card, the processor and the graphics processor address the same physical pages. A model that occupies eight gigabytes therefore occupies eight gigabytes once, and the amount of memory left for the rest of the system is simply the difference between what is installed and what the model and its caches hold.",46            "Quantization trades numerical precision for size. Weights trained as sixteen-bit floating point numbers are stored with four or eight bits each, grouped so that every group keeps its own scale and offset. The file becomes two to four times smaller and memory bandwidth, which usually limits generation speed, is used more efficiently. The price is a small loss of fidelity that grows quickly below four bits per weight.",47            "Generation proceeds one token at a time. For every new token the model attends to all previous tokens, which would be quadratic in cost if nothing were remembered. The key-value cache stores the intermediate attention state of earlier tokens so that each step only computes the newest position. The cache grows linearly with the length of the conversation and, for long contexts, can rival the size of a small model.",48            "Prompt processing and generation are measured separately because they stress different things. Reading a prompt is a batched operation: thousands of tokens flow through the network together and the throughput is high. Writing an answer is sequential and limited by the time of a single forward pass. A fair report therefore states the time to the first token, the prompt throughput and the generation throughput instead of a single average.",49            "Thermal behaviour matters on devices without fans. A phone can sustain its peak speed for a short burst, after which the system lowers the clock to keep the surface temperature acceptable. Benchmarks that last ten seconds describe the burst; benchmarks that last several minutes describe what a user experiences in a long session. Both numbers are legitimate, but they must not be confused.",50            "Retrieval augments a model with documents it was never trained on. Text is split into overlapping passages, each passage is mapped to a vector by an embedding model, and at question time the passages closest to the question are placed in the prompt. The quality of the final answer depends as much on the splitting and on the embedding model as on the generator, and every retrieved passage consumes part of the context window.",51        ]52        var out: [String] = []53        for round in 0..<3 { for (i, p) in paragraphs.enumerated() { out.append("Section \(round * paragraphs.count + i + 1). " + p) } }54        return out.joined(separator: "\n\n")55    }()56}5758public struct BenchmarkSummary: Codable, Sendable, Hashable {59    public var suiteID: String60    public var suiteVersion: Int61    public var modelID: String62    public var device: String63    public var startedAt: Date64    public var duration: TimeInterval65    public var records: [BenchmarkRecord]66    public var failures: [String: String]            // prompt label → error67    public var wasCancelled: Bool68    public var medianTokensPerSecond: Double?69    public var medianTimeToFirstToken: Double?70    public var medianPromptTokensPerSecond: Double?71    public var peakMemoryBytes: Int64?72}7374public enum BenchmarkProgress: Sendable {75    case started(index: Int, total: Int, label: String)76    case finished(BenchmarkRecord)77    case failed(label: String, error: String)78    case completed(BenchmarkSummary)                 // always last79}8081public final class BenchmarkRunner: Sendable {82    let service: any InferenceService83    let store: MetricsStore?84    let device: String8586    public init(service: any InferenceService, store: MetricsStore? = nil, device: String) {87        self.service = service; self.store = store; self.device = device88    }8990    /// Runs `suite` on one model, one prompt after the other. `loadTime` is the measured load time of the model when the caller has it.91    public func run(_ suite: BenchmarkSuite = .standard, modelID: String, base: GenerationSettings = .default, loadTime: TimeInterval? = nil) -> AsyncStream<BenchmarkProgress> {92        AsyncStream { continuation in93            let task = Task {94                let started = Date()95                let descriptor = await self.service.listModels().first { $0.id == modelID }96                var records: [BenchmarkRecord] = []; var failures: [String: String] = [:]97                for (i, p) in suite.prompts.enumerated() {98                    if Task.isCancelled { break }99                    continuation.yield(.started(index: i, total: suite.prompts.count, label: p.label))100                    var settings = base101                    settings.maxTokens = p.maxTokens; settings.seed = p.seed; settings.temperature = p.temperature102                    settings.contextLength = max(settings.contextLength, 4096)103                    var messages: [PromptMessage] = []104                    if !p.systemPrompt.isEmpty { messages.append(.system(p.systemPrompt)) }105                    messages.append(.user(p.prompt))106                    do {107                        let (_, metrics) = try await self.service.complete(modelID: modelID, request: GenerationRequest(messages: messages, settings: settings))108                        guard let metrics else { throw InferenceError.generationFailed("The backend reported no metrics.") }109                        if metrics.stopReason == .cancelled { break }110                        let record = BenchmarkRecord(modelID: modelID, modelName: descriptor?.name ?? modelID, backend: descriptor?.backend ?? .mlx,111                                                     quantization: descriptor?.quantization, device: self.device, kind: "benchmark",112                                                     promptLabel: "\(suite.id)-v\(suite.version)/\(p.id)", metrics: metrics, loadTime: i == 0 ? loadTime : nil,113                                                     contextLength: settings.contextLength)114                        try? self.store?.insert(record)115                        records.append(record)116                        continuation.yield(.finished(record))117                    } catch {118                        if Task.isCancelled { break }119                        let message = (error as? LocalizedError)?.errorDescription ?? String(describing: error)120                        failures[p.label] = message121                        continuation.yield(.failed(label: p.label, error: message))122                    }123                }124                continuation.yield(.completed(BenchmarkSummary(125                    suiteID: suite.id, suiteVersion: suite.version, modelID: modelID, device: self.device, startedAt: started, duration: Date().timeIntervalSince(started),126                    records: records, failures: failures, wasCancelled: Task.isCancelled,127                    medianTokensPerSecond: Statistics.median(records.compactMap(\.metrics.tokensPerSecond)),128                    medianTimeToFirstToken: Statistics.median(records.compactMap(\.metrics.timeToFirstToken)),129                    medianPromptTokensPerSecond: Statistics.median(records.compactMap(\.metrics.promptTokensPerSecond)),130                    peakMemoryBytes: records.compactMap(\.metrics.peakMemoryBytes).max())))131                continuation.finish()132            }133            continuation.onTermination = { _ in task.cancel() }134        }135    }136}137
Files main
Menu

Theme