SPB Git

Repository Public

MetricsStore.swift 14.1 KB 240 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatBenchmarks/Sources/MetricsStore.swift3// Description: Persistent history of measured generation metrics and arena runs, with per-model aggregates and exports. Only measured values; nil stays nil.4import Foundation5import GRDB6import LocalChatCore78public struct ModelAggregate: Codable, Sendable, Hashable, Identifiable {9    public var modelID: String10    public var modelName: String11    public var backend: BackendKind12    public var quantization: String?13    public var device: String14    public var runCount: Int15    public var medianTokensPerSecond: Double?16    public var p10TokensPerSecond: Double?17    public var p90TokensPerSecond: Double?18    public var medianTimeToFirstToken: Double?19    public var medianPromptTokensPerSecond: Double?20    public var medianLoadTime: Double?21    public var peakMemoryBytes: Int64?        // highest peak observed22    public var lastRunAt: Date23    public var id: String { modelID + "@" + device }24}2526public struct ComparisonReport: Codable, Sendable, Hashable {27    public var device: String?28    public var generatedAt: Date29    public var rows: [ModelAggregate]         // fastest median first; models without a throughput figure last3031    public func markdown() -> String {32        func n(_ v: Double?, _ f: String) -> String { v.map { String(format: f, $0) } ?? "—" }33        var out = "# Model comparison\n\nDevice: \(device ?? "all devices") · generated \(generatedAt.formatted(date: .abbreviated, time: .shortened))\n\n"34        out += "| Model | Backend | Quant. | Runs | Tokens/s (median) | p10–p90 | TTFT (s) | Prompt tokens/s | Load (s) | Peak memory |\n|---|---|---|---:|---:|---:|---:|---:|---:|---:|\n"35        for r in rows {36            let spread = r.p10TokensPerSecond != nil && r.p90TokensPerSecond != nil ? "\(n(r.p10TokensPerSecond, "%.1f"))–\(n(r.p90TokensPerSecond, "%.1f"))" : "—"37            out += "| \(r.modelName) | \(r.backend.displayName) | \(r.quantization ?? "—") | \(r.runCount) | \(n(r.medianTokensPerSecond, "%.1f")) | \(spread) | \(n(r.medianTimeToFirstToken, "%.2f")) | \(n(r.medianPromptTokensPerSecond, "%.0f")) | \(n(r.medianLoadTime, "%.2f")) | \(r.peakMemoryBytes.map(ByteFormat.memory) ?? "—") |\n"38        }39        return out40    }4142    public func csv() -> String {43        var out = "model_id,model_name,backend,quantization,device,runs,median_tokens_per_s,p10_tokens_per_s,p90_tokens_per_s,median_ttft_s,median_prompt_tokens_per_s,median_load_s,peak_memory_bytes,last_run\n"44        for r in rows {45            let fields: [String] = [r.modelID, r.modelName, r.backend.rawValue, r.quantization ?? "", r.device, "\(r.runCount)", CSV.num(r.medianTokensPerSecond),46                                    CSV.num(r.p10TokensPerSecond), CSV.num(r.p90TokensPerSecond), CSV.num(r.medianTimeToFirstToken), CSV.num(r.medianPromptTokensPerSecond),47                                    CSV.num(r.medianLoadTime), r.peakMemoryBytes.map(String.init) ?? "", ISO8601DateFormatter().string(from: r.lastRunAt)]48            out += fields.map(CSV.escape).joined(separator: ",") + "\n"49        }50        return out51    }5253    public func json() throws -> Data { try CoreCoding.prettyEncoder.encode(self) }54}5556enum CSV {57    static func escape(_ s: String) -> String {58        s.contains(where: { $0 == "," || $0 == "\"" || $0 == "\n" || $0 == "\r" }) ? "\"" + s.replacingOccurrences(of: "\"", with: "\"\"") + "\"" : s59    }60    static func num(_ v: Double?) -> String { v.map { String(format: "%.4f", $0) } ?? "" }61}6263extension CoreCoding {64    static var prettyEncoder: JSONEncoder { let e = JSONEncoder(); e.dateEncodingStrategy = .iso8601; e.outputFormatting = [.prettyPrinted, .sortedKeys]; return e }65}6667public enum Statistics {68    /// Linear-interpolated percentile (p in 0…1) of the values; nil when empty.69    public static func percentile(_ values: [Double], _ p: Double) -> Double? {70        guard !values.isEmpty else { return nil }71        let s = values.sorted(); let pos = min(max(p, 0), 1) * Double(s.count - 1)72        let lo = Int(pos.rounded(.down)), hi = Int(pos.rounded(.up))73        return s[lo] + (s[hi] - s[lo]) * (pos - Double(lo))74    }75    public static func median(_ values: [Double]) -> Double? { percentile(values, 0.5) }76}7778public final class MetricsStore: Sendable {79    let db: any DatabaseWriter8081    public init(url: URL) throws {82        try FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true)83        db = try DatabasePool(path: url.path)84        try Self.migrator.migrate(db)85    }86    private init(inMemory: Void) throws { db = try DatabaseQueue(); try Self.migrator.migrate(db) }87    public static func inMemory() throws -> MetricsStore { try MetricsStore(inMemory: ()) }8889    static var migrator: DatabaseMigrator {90        var m = DatabaseMigrator()91        m.registerMigration("v1") { db in92            try db.execute(sql: """93            CREATE TABLE record(id TEXT PRIMARY KEY NOT NULL, modelID TEXT NOT NULL, device TEXT NOT NULL, kind TEXT NOT NULL, createdAt REAL NOT NULL, json TEXT NOT NULL);94            CREATE INDEX record_model ON record(modelID, createdAt);95            CREATE INDEX record_device ON record(device);96            CREATE TABLE arena(id TEXT PRIMARY KEY NOT NULL, createdAt REAL NOT NULL, json TEXT NOT NULL);97            CREATE INDEX arena_created ON arena(createdAt);98            """)99        }100        return m101    }102103    static func encode<T: Encodable>(_ v: T) throws -> String { String(decoding: try JSONEncoder().encode(v), as: UTF8.self) }104    static func decode<T: Decodable>(_ t: T.Type, _ s: String) throws -> T { try JSONDecoder().decode(t, from: Data(s.utf8)) }105106    // MARK: Records107108    public func insert(_ record: BenchmarkRecord) throws {109        try db.write { db in110            try db.execute(sql: "INSERT OR REPLACE INTO record(id, modelID, device, kind, createdAt, json) VALUES (?,?,?,?,?,?)",111                           arguments: [record.id.uuidString, record.modelID, record.device, record.kind, record.createdAt.timeIntervalSince1970, try Self.encode(record)])112        }113    }114115    /// Records, oldest first. Every filter is optional.116    public func records(modelID: String? = nil, device: String? = nil, kind: String? = nil, since: Date? = nil, limit: Int? = nil) throws -> [BenchmarkRecord] {117        try db.read { db in118            var sql = "SELECT json FROM record WHERE 1"; var args: [any DatabaseValueConvertible] = []119            if let modelID { sql += " AND modelID = ?"; args.append(modelID) }120            if let device { sql += " AND device = ?"; args.append(device) }121            if let kind { sql += " AND kind = ?"; args.append(kind) }122            if let since { sql += " AND createdAt >= ?"; args.append(since.timeIntervalSince1970) }123            if let limit {124                sql = "SELECT json FROM (\(sql.replacingOccurrences(of: "SELECT json", with: "SELECT json, createdAt")) ORDER BY createdAt DESC LIMIT \(limit)) ORDER BY createdAt"125            } else { sql += " ORDER BY createdAt" }126            return try String.fetchAll(db, sql: sql, arguments: StatementArguments(args)).map { try Self.decode(BenchmarkRecord.self, $0) }127        }128    }129130    /// Time series of one model (oldest first), for the history charts.131    public func history(modelID: String, device: String? = nil, limit: Int? = 500) throws -> [BenchmarkRecord] {132        try records(modelID: modelID, device: device, limit: limit)133    }134135    public func devices() throws -> [String] { try db.read { try String.fetchAll($0, sql: "SELECT DISTINCT device FROM record ORDER BY device") } }136137    /// One aggregate per (model, device). Cancelled and failed runs do not count toward throughput figures.138    public func aggregates(device: String? = nil, kind: String? = nil) throws -> [ModelAggregate] {139        let all = try records(device: device, kind: kind)140        return Dictionary(grouping: all) { $0.modelID + "\u{1}" + $0.device }.values.compactMap { Self.aggregate($0) }141            .sorted { ($0.medianTokensPerSecond ?? -1, $1.modelName) > ($1.medianTokensPerSecond ?? -1, $0.modelName) }142    }143144    public func aggregate(modelID: String, device: String) throws -> ModelAggregate? { Self.aggregate(try records(modelID: modelID, device: device)) }145146    static func aggregate(_ group: [BenchmarkRecord]) -> ModelAggregate? {147        guard let last = group.max(by: { $0.createdAt < $1.createdAt }) else { return nil }148        let valid = group.filter { $0.metrics.stopReason != .cancelled && $0.metrics.stopReason != .error }149        let tps = valid.compactMap(\.metrics.tokensPerSecond)150        return ModelAggregate(151            modelID: last.modelID, modelName: last.modelName, backend: last.backend, quantization: last.quantization, device: last.device, runCount: group.count,152            medianTokensPerSecond: Statistics.median(tps), p10TokensPerSecond: Statistics.percentile(tps, 0.1), p90TokensPerSecond: Statistics.percentile(tps, 0.9),153            medianTimeToFirstToken: Statistics.median(valid.compactMap(\.metrics.timeToFirstToken)),154            medianPromptTokensPerSecond: Statistics.median(valid.compactMap(\.metrics.promptTokensPerSecond)),155            medianLoadTime: Statistics.median(group.compactMap(\.loadTime)), peakMemoryBytes: group.compactMap(\.metrics.peakMemoryBytes).max(), lastRunAt: last.createdAt)156    }157158    /// Median measured tokens/s per model id on `device` (input of the router and of recommendations).159    public func measuredSpeed(device: String) throws -> [String: Double] {160        var out: [String: Double] = [:]161        for a in try aggregates(device: device) { if let v = a.medianTokensPerSecond { out[a.modelID] = v } }162        return out163    }164165    public func comparisonReport(modelIDs: [String]? = nil, device: String? = nil) throws -> ComparisonReport {166        var rows = try aggregates(device: device)167        if let modelIDs { rows = rows.filter { modelIDs.contains($0.modelID) } }168        return ComparisonReport(device: device, generatedAt: Date(), rows: rows)169    }170171    public func deleteRecord(_ id: UUID) throws { try db.write { try $0.execute(sql: "DELETE FROM record WHERE id = ?", arguments: [id.uuidString]) } }172    public func deleteRecords(modelID: String) throws { try db.write { try $0.execute(sql: "DELETE FROM record WHERE modelID = ?", arguments: [modelID]) } }173174    /// Deletes records older than `date`. Returns how many were removed.175    @discardableResult176    public func prune(olderThan date: Date) throws -> Int {177        try db.write { db in178            try db.execute(sql: "DELETE FROM record WHERE createdAt < ?", arguments: [date.timeIntervalSince1970])179            return db.changesCount180        }181    }182183    public func exportRecordsCSV(modelID: String? = nil, device: String? = nil) throws -> String {184        var out = "id,created_at,model_id,model_name,backend,quantization,device,kind,prompt_label,prompt_tokens,cached_prompt_tokens,completion_tokens,reasoning_tokens,ttft_s,prompt_time_s,generation_time_s,total_time_s,tokens_per_s,prompt_tokens_per_s,peak_memory_bytes,active_memory_bytes,load_time_s,context_length,stop_reason\n"185        func i(_ v: Int?) -> String { v.map(String.init) ?? "" }186        let iso = ISO8601DateFormatter()187        for r in try records(modelID: modelID, device: device) {188            let m = r.metrics189            let fields: [String] = [r.id.uuidString, iso.string(from: r.createdAt), r.modelID, r.modelName, r.backend.rawValue, r.quantization ?? "", r.device, r.kind,190                                    r.promptLabel ?? "", i(m.promptTokens), i(m.cachedPromptTokens), i(m.completionTokens), i(m.reasoningTokens), CSV.num(m.timeToFirstToken),191                                    CSV.num(m.promptTime), CSV.num(m.generationTime), CSV.num(m.totalTime), CSV.num(m.tokensPerSecond), CSV.num(m.promptTokensPerSecond),192                                    m.peakMemoryBytes.map(String.init) ?? "", m.activeMemoryBytes.map(String.init) ?? "", CSV.num(r.loadTime), i(r.contextLength), m.stopReason.rawValue]193            out += fields.map(CSV.escape).joined(separator: ",") + "\n"194        }195        return out196    }197198    public func exportRecordsJSON(modelID: String? = nil, device: String? = nil) throws -> Data {199        try CoreCoding.prettyEncoder.encode(try records(modelID: modelID, device: device))200    }201202    // MARK: Arena runs203204    public func saveArenaRun(_ run: ArenaRun) throws {205        try db.write { db in206            try db.execute(sql: "INSERT OR REPLACE INTO arena(id, createdAt, json) VALUES (?,?,?)",207                           arguments: [run.id.uuidString, run.createdAt.timeIntervalSince1970, try Self.encode(run)])208        }209    }210211    /// Newest first.212    public func arenaRuns(limit: Int? = nil) throws -> [ArenaRun] {213        try db.read { db in214            try String.fetchAll(db, sql: "SELECT json FROM arena ORDER BY createdAt DESC" + (limit.map { " LIMIT \($0)" } ?? "")).map { try Self.decode(ArenaRun.self, $0) }215        }216    }217218    public func arenaRun(_ id: UUID) throws -> ArenaRun? {219        try db.read { try String.fetchOne($0, sql: "SELECT json FROM arena WHERE id = ?", arguments: [id.uuidString]).map { try Self.decode(ArenaRun.self, $0) } }220    }221222    public func deleteArenaRun(_ id: UUID) throws { try db.write { try $0.execute(sql: "DELETE FROM arena WHERE id = ?", arguments: [id.uuidString]) } }223224    /// Appends the user's verdict to a stored run and returns the updated run.225    @discardableResult226    public func recordVote(_ verdict: ArenaVerdict, runID: UUID) throws -> ArenaRun {227        try db.write { db in228            guard let json = try String.fetchOne(db, sql: "SELECT json FROM arena WHERE id = ?", arguments: [runID.uuidString]) else { throw ArenaError.runNotFound }229            var run = try Self.decode(ArenaRun.self, json)230            if case .winner(let id) = verdict, !run.entries.contains(where: { $0.contestant.id == id }) { throw ArenaError.unknownContestant }231            run.votes.append(ArenaVote(verdict: verdict))232            try db.execute(sql: "UPDATE arena SET json = ? WHERE id = ?", arguments: [try Self.encode(run), runID.uuidString])233            return run234        }235    }236237    /// Leaderboard over every stored vote. See `ArenaLeaderboard` for the method.238    public func leaderboard() throws -> [LeaderboardRow] { ArenaLeaderboard.compute(try arenaRuns().reversed()) }239}240
Files main
Menu

Theme