SPB Git

Repository Public MIT

LocalBenchmark.swift 9.3 KB 181 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Packages/AstraLocal/Sources/LocalBenchmark.swift10// Module           : Packages11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation9091/// Result of one benchmark run. Plain data: the caller decides whether and where to keep it.92public struct LocalBenchmarkResult: Sendable, Hashable, Codable {93    public struct Sample: Sendable, Hashable, Codable {94        public var promptID: String95        /// True for the first generation after the model was loaded: it pays for shader compilation and page-ins that later ones do not.96        public var cold: Bool97        public var timeToFirstToken: TimeInterval?98        public var decodeTokensPerSecond: Double?99        public var promptTokensPerSecond: Double?100        public var promptTokens: Int?101        public var outputTokens: Int?102        public var peakMemoryBytes: Int64?103        public var totalTime: TimeInterval104        public var stopReason: StopReason105        public var thermalBefore: ThermalLevel?106        public var thermalAfter: ThermalLevel?107        /// Beginning of the generated text, to show that real text came out.108        public var outputPreview: String109    }110    public var modelID: String111    public var modelRevision: String?112    public var device: DeviceFacts113    public var startedAt: Date114    public var loadTime: TimeInterval?115    public var weightsMemoryBytes: Int64?116    public var thermalBefore: ThermalLevel117    public var thermalAfter: ThermalLevel118    public var samples: [Sample]119120    public var cold: Sample? { samples.first { $0.cold } }121    public var warm: [Sample] { samples.filter { !$0.cold } }122    /// Median over the warm samples; nil when there are none.123    public var warmDecodeTokensPerSecond: Double? { Self.median(warm.compactMap(\.decodeTokensPerSecond)) }124    public var warmTimeToFirstToken: TimeInterval? { Self.median(warm.compactMap(\.timeToFirstToken)) }125    public var peakMemoryBytes: Int64? { samples.compactMap(\.peakMemoryBytes).max() }126127    static func median(_ values: [Double]) -> Double? {128        guard !values.isEmpty else { return nil }129        let s = values.sorted()130        return s.count % 2 == 1 ? s[s.count / 2] : (s[s.count / 2 - 1] + s[s.count / 2]) / 2131    }132}133134/// Runs a fixed prompt set against a model through the engine and records what was measured.135public struct LocalBenchmark: Sendable {136    public struct Prompt: Sendable, Hashable {137        public var id: String138        public var system: String?139        public var text: String140        public init(id: String, system: String? = nil, text: String) { self.id = id; self.system = system; self.text = text }141    }142143    /// Short, medium and long-prompt cases. Fixed so runs are comparable across models and devices.144    public static let standardPrompts: [Prompt] = [145        Prompt(id: "short-answer", text: "Explain in two sentences why the sky is blue."),146        Prompt(id: "list", text: "List five practical tips for taking better photos with a phone, one line each."),147        Prompt(id: "summarize", system: "You are a concise assistant.",148               text: "Summarize the following in three bullet points:\n\n" + String(repeating: "Local language models run entirely on the device, so prompts never leave it. They trade raw capability for privacy, availability without a network, and predictable cost. Their speed depends on memory bandwidth and on how many tokens are already in the context. ", count: 6)),149        Prompt(id: "reasoning", text: "A train leaves at 9:40 and arrives at 11:05. How long is the trip? Answer with the duration only."),150    ]151152    public var prompts: [Prompt]153    public var maxOutputTokens: Int154    public init(prompts: [Prompt] = LocalBenchmark.standardPrompts, maxOutputTokens: Int = 160) { self.prompts = prompts; self.maxOutputTokens = maxOutputTokens }155156    /// The model must be loadable by `engine`. Pass `reload: true` to unload it first so the first sample is a true cold start.157    public func run(modelID: String, engine: LocalEngine, revision: String? = nil, reload: Bool = false) async throws -> LocalBenchmarkResult {158        if reload { await engine.unload(modelID) }159        let info = try await engine.load(modelID)160        let thermalBefore = ThermalLevel.current()161        let started = Date()162        var samples: [LocalBenchmarkResult.Sample] = []163        for prompt in prompts {164            try Task.checkCancellation()165            let request = GenerationRequest(model: ModelRef(provider: .mlx, modelID: modelID), system: prompt.system, messages: [.user(prompt.text)],166                                            parameters: GenerationParameters(temperature: 0, maxOutputTokens: maxOutputTokens, seed: 7))167            var text = ""168            for try await event in engine.generate(request) { if case .textDelta(let t) = event { text += t } }169            // the engine records metrics before it finishes the stream170            guard let m = engine.monitor.lastMetrics, m.modelID == modelID else { throw AstraError.local("The runtime reported no metrics for this generation.") }171            samples.append(.init(promptID: prompt.id, cold: m.isColdStart, timeToFirstToken: m.timeToFirstToken, decodeTokensPerSecond: m.decodeTokensPerSecond,172                                 promptTokensPerSecond: m.promptTokensPerSecond, promptTokens: m.promptTokens, outputTokens: m.completionTokens,173                                 peakMemoryBytes: m.peakMemoryBytes, totalTime: m.totalTime, stopReason: m.stopReason,174                                 thermalBefore: m.thermalBefore, thermalAfter: m.thermalAfter, outputPreview: String(text.prefix(240))))175        }176        return LocalBenchmarkResult(modelID: modelID, modelRevision: revision ?? engine.store.card(id: modelID)?.revision, device: .current(), startedAt: started,177                                    loadTime: info.loadTime, weightsMemoryBytes: info.weightsMemoryBytes, thermalBefore: thermalBefore, thermalAfter: .current(),178                                    samples: samples)179    }180}181
Files main
Menu

Theme