Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatBenchmarks/Sources/MetricsStore.swift3// Description: Persistent history of measured generation metrics and arena runs, with per-model aggregates and exports. Only measured values; nil stays nil.4import Foundation5import GRDB6import LocalChatCore78public struct ModelAggregate: Codable, Sendable, Hashable, Identifiable {9 public var modelID: String10 public var modelName: String11 public var backend: BackendKind12 public var quantization: String?13 public var device: String14 public var runCount: Int15 public var medianTokensPerSecond: Double?16 public var p10TokensPerSecond: Double?17 public var p90TokensPerSecond: Double?18 public var medianTimeToFirstToken: Double?19 public var medianPromptTokensPerSecond: Double?20 public var medianLoadTime: Double?21 public var peakMemoryBytes: Int64? // highest peak observed22 public var lastRunAt: Date23 public var id: String { modelID + "@" + device }24}2526public struct ComparisonReport: Codable, Sendable, Hashable {27 public var device: String?28 public var generatedAt: Date29 public var rows: [ModelAggregate] // fastest median first; models without a throughput figure last3031 public func markdown() -> String {32 func n(_ v: Double?, _ f: String) -> String { v.map { String(format: f, $0) } ?? "—" }33 var out = "# Model comparison\n\nDevice: \(device ?? "all devices") · generated \(generatedAt.formatted(date: .abbreviated, time: .shortened))\n\n"34 out += "| Model | Backend | Quant. | Runs | Tokens/s (median) | p10–p90 | TTFT (s) | Prompt tokens/s | Load (s) | Peak memory |\n|---|---|---|---:|---:|---:|---:|---:|---:|---:|\n"35 for r in rows {36 let spread = r.p10TokensPerSecond != nil && r.p90TokensPerSecond != nil ? "\(n(r.p10TokensPerSecond, "%.1f"))–\(n(r.p90TokensPerSecond, "%.1f"))" : "—"37 out += "| \(r.modelName) | \(r.backend.displayName) | \(r.quantization ?? "—") | \(r.runCount) | \(n(r.medianTokensPerSecond, "%.1f")) | \(spread) | \(n(r.medianTimeToFirstToken, "%.2f")) | \(n(r.medianPromptTokensPerSecond, "%.0f")) | \(n(r.medianLoadTime, "%.2f")) | \(r.peakMemoryBytes.map(ByteFormat.memory) ?? "—") |\n"38 }39 return out40 }4142 public func csv() -> String {43 var out = "model_id,model_name,backend,quantization,device,runs,median_tokens_per_s,p10_tokens_per_s,p90_tokens_per_s,median_ttft_s,median_prompt_tokens_per_s,median_load_s,peak_memory_bytes,last_run\n"44 for r in rows {45 let fields: [String] = [r.modelID, r.modelName, r.backend.rawValue, r.quantization ?? "", r.device, "\(r.runCount)", CSV.num(r.medianTokensPerSecond),46 CSV.num(r.p10TokensPerSecond), CSV.num(r.p90TokensPerSecond), CSV.num(r.medianTimeToFirstToken), CSV.num(r.medianPromptTokensPerSecond),47 CSV.num(r.medianLoadTime), r.peakMemoryBytes.map(String.init) ?? "", ISO8601DateFormatter().string(from: r.lastRunAt)]48 out += fields.map(CSV.escape).joined(separator: ",") + "\n"49 }50 return out51 }5253 public func json() throws -> Data { try CoreCoding.prettyEncoder.encode(self) }54}5556enum CSV {57 static func escape(_ s: String) -> String {58 s.contains(where: { $0 == "," || $0 == "\"" || $0 == "\n" || $0 == "\r" }) ? "\"" + s.replacingOccurrences(of: "\"", with: "\"\"") + "\"" : s59 }60 static func num(_ v: Double?) -> String { v.map { String(format: "%.4f", $0) } ?? "" }61}6263extension CoreCoding {64 static var prettyEncoder: JSONEncoder { let e = JSONEncoder(); e.dateEncodingStrategy = .iso8601; e.outputFormatting = [.prettyPrinted, .sortedKeys]; return e }65}6667public enum Statistics {68 /// Linear-interpolated percentile (p in 0…1) of the values; nil when empty.69 public static func percentile(_ values: [Double], _ p: Double) -> Double? {70 guard !values.isEmpty else { return nil }71 let s = values.sorted(); let pos = min(max(p, 0), 1) * Double(s.count - 1)72 let lo = Int(pos.rounded(.down)), hi = Int(pos.rounded(.up))73 return s[lo] + (s[hi] - s[lo]) * (pos - Double(lo))74 }75 public static func median(_ values: [Double]) -> Double? { percentile(values, 0.5) }76}7778public final class MetricsStore: Sendable {79 let db: any DatabaseWriter8081 public init(url: URL) throws {82 try FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true)83 db = try DatabasePool(path: url.path)84 try Self.migrator.migrate(db)85 }86 private init(inMemory: Void) throws { db = try DatabaseQueue(); try Self.migrator.migrate(db) }87 public static func inMemory() throws -> MetricsStore { try MetricsStore(inMemory: ()) }8889 static var migrator: DatabaseMigrator {90 var m = DatabaseMigrator()91 m.registerMigration("v1") { db in92 try db.execute(sql: """93 CREATE TABLE record(id TEXT PRIMARY KEY NOT NULL, modelID TEXT NOT NULL, device TEXT NOT NULL, kind TEXT NOT NULL, createdAt REAL NOT NULL, json TEXT NOT NULL);94 CREATE INDEX record_model ON record(modelID, createdAt);95 CREATE INDEX record_device ON record(device);96 CREATE TABLE arena(id TEXT PRIMARY KEY NOT NULL, createdAt REAL NOT NULL, json TEXT NOT NULL);97 CREATE INDEX arena_created ON arena(createdAt);98 """)99 }100 return m101 }102103 static func encode<T: Encodable>(_ v: T) throws -> String { String(decoding: try JSONEncoder().encode(v), as: UTF8.self) }104 static func decode<T: Decodable>(_ t: T.Type, _ s: String) throws -> T { try JSONDecoder().decode(t, from: Data(s.utf8)) }105106 // MARK: Records107108 public func insert(_ record: BenchmarkRecord) throws {109 try db.write { db in110 try db.execute(sql: "INSERT OR REPLACE INTO record(id, modelID, device, kind, createdAt, json) VALUES (?,?,?,?,?,?)",111 arguments: [record.id.uuidString, record.modelID, record.device, record.kind, record.createdAt.timeIntervalSince1970, try Self.encode(record)])112 }113 }114115 /// Records, oldest first. Every filter is optional.116 public func records(modelID: String? = nil, device: String? = nil, kind: String? = nil, since: Date? = nil, limit: Int? = nil) throws -> [BenchmarkRecord] {117 try db.read { db in118 var sql = "SELECT json FROM record WHERE 1"; var args: [any DatabaseValueConvertible] = []119 if let modelID { sql += " AND modelID = ?"; args.append(modelID) }120 if let device { sql += " AND device = ?"; args.append(device) }121 if let kind { sql += " AND kind = ?"; args.append(kind) }122 if let since { sql += " AND createdAt >= ?"; args.append(since.timeIntervalSince1970) }123 if let limit {124 sql = "SELECT json FROM (\(sql.replacingOccurrences(of: "SELECT json", with: "SELECT json, createdAt")) ORDER BY createdAt DESC LIMIT \(limit)) ORDER BY createdAt"125 } else { sql += " ORDER BY createdAt" }126 return try String.fetchAll(db, sql: sql, arguments: StatementArguments(args)).map { try Self.decode(BenchmarkRecord.self, $0) }127 }128 }129130 /// Time series of one model (oldest first), for the history charts.131 public func history(modelID: String, device: String? = nil, limit: Int? = 500) throws -> [BenchmarkRecord] {132 try records(modelID: modelID, device: device, limit: limit)133 }134135 public func devices() throws -> [String] { try db.read { try String.fetchAll($0, sql: "SELECT DISTINCT device FROM record ORDER BY device") } }136137 /// One aggregate per (model, device). Cancelled and failed runs do not count toward throughput figures.138 public func aggregates(device: String? = nil, kind: String? = nil) throws -> [ModelAggregate] {139 let all = try records(device: device, kind: kind)140 return Dictionary(grouping: all) { $0.modelID + "\u{1}" + $0.device }.values.compactMap { Self.aggregate($0) }141 .sorted { ($0.medianTokensPerSecond ?? -1, $1.modelName) > ($1.medianTokensPerSecond ?? -1, $0.modelName) }142 }143144 public func aggregate(modelID: String, device: String) throws -> ModelAggregate? { Self.aggregate(try records(modelID: modelID, device: device)) }145146 static func aggregate(_ group: [BenchmarkRecord]) -> ModelAggregate? {147 guard let last = group.max(by: { $0.createdAt < $1.createdAt }) else { return nil }148 let valid = group.filter { $0.metrics.stopReason != .cancelled && $0.metrics.stopReason != .error }149 let tps = valid.compactMap(\.metrics.tokensPerSecond)150 return ModelAggregate(151 modelID: last.modelID, modelName: last.modelName, backend: last.backend, quantization: last.quantization, device: last.device, runCount: group.count,152 medianTokensPerSecond: Statistics.median(tps), p10TokensPerSecond: Statistics.percentile(tps, 0.1), p90TokensPerSecond: Statistics.percentile(tps, 0.9),153 medianTimeToFirstToken: Statistics.median(valid.compactMap(\.metrics.timeToFirstToken)),154 medianPromptTokensPerSecond: Statistics.median(valid.compactMap(\.metrics.promptTokensPerSecond)),155 medianLoadTime: Statistics.median(group.compactMap(\.loadTime)), peakMemoryBytes: group.compactMap(\.metrics.peakMemoryBytes).max(), lastRunAt: last.createdAt)156 }157158 /// Median measured tokens/s per model id on `device` (input of the router and of recommendations).159 public func measuredSpeed(device: String) throws -> [String: Double] {160 var out: [String: Double] = [:]161 for a in try aggregates(device: device) { if let v = a.medianTokensPerSecond { out[a.modelID] = v } }162 return out163 }164165 public func comparisonReport(modelIDs: [String]? = nil, device: String? = nil) throws -> ComparisonReport {166 var rows = try aggregates(device: device)167 if let modelIDs { rows = rows.filter { modelIDs.contains($0.modelID) } }168 return ComparisonReport(device: device, generatedAt: Date(), rows: rows)169 }170171 public func deleteRecord(_ id: UUID) throws { try db.write { try $0.execute(sql: "DELETE FROM record WHERE id = ?", arguments: [id.uuidString]) } }172 public func deleteRecords(modelID: String) throws { try db.write { try $0.execute(sql: "DELETE FROM record WHERE modelID = ?", arguments: [modelID]) } }173174 /// Deletes records older than `date`. Returns how many were removed.175 @discardableResult176 public func prune(olderThan date: Date) throws -> Int {177 try db.write { db in178 try db.execute(sql: "DELETE FROM record WHERE createdAt < ?", arguments: [date.timeIntervalSince1970])179 return db.changesCount180 }181 }182183 public func exportRecordsCSV(modelID: String? = nil, device: String? = nil) throws -> String {184 var out = "id,created_at,model_id,model_name,backend,quantization,device,kind,prompt_label,prompt_tokens,cached_prompt_tokens,completion_tokens,reasoning_tokens,ttft_s,prompt_time_s,generation_time_s,total_time_s,tokens_per_s,prompt_tokens_per_s,peak_memory_bytes,active_memory_bytes,load_time_s,context_length,stop_reason\n"185 func i(_ v: Int?) -> String { v.map(String.init) ?? "" }186 let iso = ISO8601DateFormatter()187 for r in try records(modelID: modelID, device: device) {188 let m = r.metrics189 let fields: [String] = [r.id.uuidString, iso.string(from: r.createdAt), r.modelID, r.modelName, r.backend.rawValue, r.quantization ?? "", r.device, r.kind,190 r.promptLabel ?? "", i(m.promptTokens), i(m.cachedPromptTokens), i(m.completionTokens), i(m.reasoningTokens), CSV.num(m.timeToFirstToken),191 CSV.num(m.promptTime), CSV.num(m.generationTime), CSV.num(m.totalTime), CSV.num(m.tokensPerSecond), CSV.num(m.promptTokensPerSecond),192 m.peakMemoryBytes.map(String.init) ?? "", m.activeMemoryBytes.map(String.init) ?? "", CSV.num(r.loadTime), i(r.contextLength), m.stopReason.rawValue]193 out += fields.map(CSV.escape).joined(separator: ",") + "\n"194 }195 return out196 }197198 public func exportRecordsJSON(modelID: String? = nil, device: String? = nil) throws -> Data {199 try CoreCoding.prettyEncoder.encode(try records(modelID: modelID, device: device))200 }201202 // MARK: Arena runs203204 public func saveArenaRun(_ run: ArenaRun) throws {205 try db.write { db in206 try db.execute(sql: "INSERT OR REPLACE INTO arena(id, createdAt, json) VALUES (?,?,?)",207 arguments: [run.id.uuidString, run.createdAt.timeIntervalSince1970, try Self.encode(run)])208 }209 }210211 /// Newest first.212 public func arenaRuns(limit: Int? = nil) throws -> [ArenaRun] {213 try db.read { db in214 try String.fetchAll(db, sql: "SELECT json FROM arena ORDER BY createdAt DESC" + (limit.map { " LIMIT \($0)" } ?? "")).map { try Self.decode(ArenaRun.self, $0) }215 }216 }217218 public func arenaRun(_ id: UUID) throws -> ArenaRun? {219 try db.read { try String.fetchOne($0, sql: "SELECT json FROM arena WHERE id = ?", arguments: [id.uuidString]).map { try Self.decode(ArenaRun.self, $0) } }220 }221222 public func deleteArenaRun(_ id: UUID) throws { try db.write { try $0.execute(sql: "DELETE FROM arena WHERE id = ?", arguments: [id.uuidString]) } }223224 /// Appends the user's verdict to a stored run and returns the updated run.225 @discardableResult226 public func recordVote(_ verdict: ArenaVerdict, runID: UUID) throws -> ArenaRun {227 try db.write { db in228 guard let json = try String.fetchOne(db, sql: "SELECT json FROM arena WHERE id = ?", arguments: [runID.uuidString]) else { throw ArenaError.runNotFound }229 var run = try Self.decode(ArenaRun.self, json)230 if case .winner(let id) = verdict, !run.entries.contains(where: { $0.contestant.id == id }) { throw ArenaError.unknownContestant }231 run.votes.append(ArenaVote(verdict: verdict))232 try db.execute(sql: "UPDATE arena SET json = ? WHERE id = ?", arguments: [try Self.encode(run), runID.uuidString])233 return run234 }235 }236237 /// Leaderboard over every stored vote. See `ArenaLeaderboard` for the method.238 public func leaderboard() throws -> [LeaderboardRow] { ArenaLeaderboard.compute(try arenaRuns().reversed()) }239}240