Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatBenchmarks/Sources/BenchmarkSuite.swift3// Description: A fixed, versioned benchmark suite and its runner. Every figure comes from the backend's own measurements.4import Foundation5import LocalChatCore67public struct BenchmarkPrompt: Codable, Sendable, Hashable, Identifiable {8 public var id: String9 public var label: String10 public var purpose: String11 public var systemPrompt: String12 public var prompt: String13 public var maxTokens: Int14 public var seed: UInt6415 public var temperature: Double16}1718public struct BenchmarkSuite: Codable, Sendable, Hashable, Identifiable {19 public var id: String20 public var version: Int21 public var name: String22 public var prompts: [BenchmarkPrompt]2324 /// Changing any prompt, seed or limit requires bumping `version`: results are only comparable within one version.25 public static let standard = BenchmarkSuite(id: "standard", version: 1, name: "Standard suite", prompts: [26 BenchmarkPrompt(id: "short-answer", label: "Short answer", purpose: "Time to first token and latency of a brief reply.",27 systemPrompt: "You are a concise assistant.", prompt: "In one sentence, what is the capital of Canada and on which river does it lie?",28 maxTokens: 64, seed: 11, temperature: 0),29 BenchmarkPrompt(id: "long-generation", label: "Long generation", purpose: "Sustained generation throughput.",30 systemPrompt: "You are a clear technical writer.",31 prompt: "Write a detailed, well-structured explanation (about 500 words) of how a hash table works, covering hashing, collisions, load factor and resizing.",32 maxTokens: 700, seed: 23, temperature: 0.7),33 BenchmarkPrompt(id: "long-prompt", label: "Long prompt ingestion", purpose: "Prompt processing throughput on a prompt of roughly two thousand tokens.",34 systemPrompt: "You are a careful reader.", prompt: longPassage + "\n\nList the three most important points of the text above as short bullet points.",35 maxTokens: 160, seed: 37, temperature: 0),36 BenchmarkPrompt(id: "code", label: "Code", purpose: "Code generation throughput.",37 systemPrompt: "You are a senior software engineer. Answer with code first.",38 prompt: "Write a Swift function that merges two sorted arrays of integers into one sorted array without using sort(), then give two test cases.",39 maxTokens: 400, seed: 41, temperature: 0.2),40 ])4142 /// Deterministic passage (≈ 8 000 characters, about two thousand tokens with common tokenizers; the exact count is measured, never assumed).43 static let longPassage: String = {44 let paragraphs = [45 "Unified memory changes how a language model is deployed on a personal computer. Instead of copying weights from system memory to a separate graphics card, the processor and the graphics processor address the same physical pages. A model that occupies eight gigabytes therefore occupies eight gigabytes once, and the amount of memory left for the rest of the system is simply the difference between what is installed and what the model and its caches hold.",46 "Quantization trades numerical precision for size. Weights trained as sixteen-bit floating point numbers are stored with four or eight bits each, grouped so that every group keeps its own scale and offset. The file becomes two to four times smaller and memory bandwidth, which usually limits generation speed, is used more efficiently. The price is a small loss of fidelity that grows quickly below four bits per weight.",47 "Generation proceeds one token at a time. For every new token the model attends to all previous tokens, which would be quadratic in cost if nothing were remembered. The key-value cache stores the intermediate attention state of earlier tokens so that each step only computes the newest position. The cache grows linearly with the length of the conversation and, for long contexts, can rival the size of a small model.",48 "Prompt processing and generation are measured separately because they stress different things. Reading a prompt is a batched operation: thousands of tokens flow through the network together and the throughput is high. Writing an answer is sequential and limited by the time of a single forward pass. A fair report therefore states the time to the first token, the prompt throughput and the generation throughput instead of a single average.",49 "Thermal behaviour matters on devices without fans. A phone can sustain its peak speed for a short burst, after which the system lowers the clock to keep the surface temperature acceptable. Benchmarks that last ten seconds describe the burst; benchmarks that last several minutes describe what a user experiences in a long session. Both numbers are legitimate, but they must not be confused.",50 "Retrieval augments a model with documents it was never trained on. Text is split into overlapping passages, each passage is mapped to a vector by an embedding model, and at question time the passages closest to the question are placed in the prompt. The quality of the final answer depends as much on the splitting and on the embedding model as on the generator, and every retrieved passage consumes part of the context window.",51 ]52 var out: [String] = []53 for round in 0..<3 { for (i, p) in paragraphs.enumerated() { out.append("Section \(round * paragraphs.count + i + 1). " + p) } }54 return out.joined(separator: "\n\n")55 }()56}5758public struct BenchmarkSummary: Codable, Sendable, Hashable {59 public var suiteID: String60 public var suiteVersion: Int61 public var modelID: String62 public var device: String63 public var startedAt: Date64 public var duration: TimeInterval65 public var records: [BenchmarkRecord]66 public var failures: [String: String] // prompt label → error67 public var wasCancelled: Bool68 public var medianTokensPerSecond: Double?69 public var medianTimeToFirstToken: Double?70 public var medianPromptTokensPerSecond: Double?71 public var peakMemoryBytes: Int64?72}7374public enum BenchmarkProgress: Sendable {75 case started(index: Int, total: Int, label: String)76 case finished(BenchmarkRecord)77 case failed(label: String, error: String)78 case completed(BenchmarkSummary) // always last79}8081public final class BenchmarkRunner: Sendable {82 let service: any InferenceService83 let store: MetricsStore?84 let device: String8586 public init(service: any InferenceService, store: MetricsStore? = nil, device: String) {87 self.service = service; self.store = store; self.device = device88 }8990 /// Runs `suite` on one model, one prompt after the other. `loadTime` is the measured load time of the model when the caller has it.91 public func run(_ suite: BenchmarkSuite = .standard, modelID: String, base: GenerationSettings = .default, loadTime: TimeInterval? = nil) -> AsyncStream<BenchmarkProgress> {92 AsyncStream { continuation in93 let task = Task {94 let started = Date()95 let descriptor = await self.service.listModels().first { $0.id == modelID }96 var records: [BenchmarkRecord] = []; var failures: [String: String] = [:]97 for (i, p) in suite.prompts.enumerated() {98 if Task.isCancelled { break }99 continuation.yield(.started(index: i, total: suite.prompts.count, label: p.label))100 var settings = base101 settings.maxTokens = p.maxTokens; settings.seed = p.seed; settings.temperature = p.temperature102 settings.contextLength = max(settings.contextLength, 4096)103 var messages: [PromptMessage] = []104 if !p.systemPrompt.isEmpty { messages.append(.system(p.systemPrompt)) }105 messages.append(.user(p.prompt))106 do {107 let (_, metrics) = try await self.service.complete(modelID: modelID, request: GenerationRequest(messages: messages, settings: settings))108 guard let metrics else { throw InferenceError.generationFailed("The backend reported no metrics.") }109 if metrics.stopReason == .cancelled { break }110 let record = BenchmarkRecord(modelID: modelID, modelName: descriptor?.name ?? modelID, backend: descriptor?.backend ?? .mlx,111 quantization: descriptor?.quantization, device: self.device, kind: "benchmark",112 promptLabel: "\(suite.id)-v\(suite.version)/\(p.id)", metrics: metrics, loadTime: i == 0 ? loadTime : nil,113 contextLength: settings.contextLength)114 try? self.store?.insert(record)115 records.append(record)116 continuation.yield(.finished(record))117 } catch {118 if Task.isCancelled { break }119 let message = (error as? LocalizedError)?.errorDescription ?? String(describing: error)120 failures[p.label] = message121 continuation.yield(.failed(label: p.label, error: message))122 }123 }124 continuation.yield(.completed(BenchmarkSummary(125 suiteID: suite.id, suiteVersion: suite.version, modelID: modelID, device: self.device, startedAt: started, duration: Date().timeIntervalSince(started),126 records: records, failures: failures, wasCancelled: Task.isCancelled,127 medianTokensPerSecond: Statistics.median(records.compactMap(\.metrics.tokensPerSecond)),128 medianTimeToFirstToken: Statistics.median(records.compactMap(\.metrics.timeToFirstToken)),129 medianPromptTokensPerSecond: Statistics.median(records.compactMap(\.metrics.promptTokensPerSecond)),130 peakMemoryBytes: records.compactMap(\.metrics.peakMemoryBytes).max())))131 continuation.finish()132 }133 continuation.onTermination = { _ in task.cancel() }134 }135 }136}137