Repository Public MIT
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Astra9// File : Packages/AstraUI/Sources/Lab/BenchmarksLab.swift10// Module : Lab11// Repository : https://git.spboucher.ai/orix-astra12// Created : 2026-11-2113// Last Modified : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author : Simon-Pierre Boucher20// Position : Professor of Finance21// Institution : Université du Québec en Outaouais (UQO)22// Department : Département des sciences administratives23// Organization : SP Boucher24//25// Website : https://spboucher.ai26// Git : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General : info@spboucher.ai33// Professional : contact@spboucher.ai34// Direct : simon@spboucher.ai35// Research : research@spboucher.ai36// Development : dev@spboucher.ai37// Support : support@spboucher.ai38// Security/Admin : admin@spboucher.ai39// Notifications : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification : Public66// Distribution : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License : Proprietary, all rights reserved (see license file)80// License File : LICENSE81// Maintainer : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import SwiftUI89import Charts90import AstraCore91import AstraDesign92import AstraLocal93import AstraPersistence9495/// Benchmarks: a fixed prompt set on an installed on-device model (cold and warm kept apart), and a tiny latency probe for cloud models.96struct BenchmarksLabView: View {97 @Environment(AppModel.self) private var app98 @State private var localID: String?99 @State private var reload = true100 @State private var isRunning = false101 @State private var progress = 0102 @State private var result: LocalBenchmarkResult?103 @State private var localError: AstraError?104 @State private var localTask: Task<Void, Never>?105106 @State private var probeModel: ModelRef?107 @State private var probe: LabRunController?108 @State private var showPicker = false109110 @State private var records: [BenchmarkRecord]?111 @State private var historyError: String?112 @State private var exported: ExportedFile?113 @State private var confirmClear = false114115 private let promptCount = LocalBenchmark.standardPrompts.count116117 var body: some View {118 LabPage(title: "Benchmarks") {119 localSection120 probeSection121 historySection122 }123 .toolbar {124 ToolbarItem(placement: .topBarTrailing) {125 Menu {126 Button { export(.csv) } label: { Label("Export CSV", systemImage: "tablecells") }127 Button { export(.json) } label: { Label("Export JSON", systemImage: "curlybraces") }128 Button { export(.markdown) } label: { Label("Export Markdown", systemImage: "doc.plaintext") }129 Divider()130 Button(role: .destructive) { confirmClear = true } label: { Label("Delete all records", systemImage: "trash") }131 } label: { Label("More", systemImage: "ellipsis.circle") }132 .disabled(records?.isEmpty ?? true)133 .accessibilityIdentifier("benchmarks-menu")134 }135 }136 .task {137 if localID == nil { localID = app.local.installed.first?.id }138 for await _ in app.store.changes(in: [.benchmark]) { await loadHistory() }139 }140 .sheet(isPresented: $showPicker) { ModelPickerSheet(selected: probeModel, title: "Model to probe") { probeModel = $0 } }141 .sheet(item: $exported) { ShareSheet(items: [$0.url]) }142 .onChange(of: probe?.phase) { _, phase in if phase == .done { saveProbe() } }143 .confirmationDialog("Delete every benchmark record?", isPresented: $confirmClear, titleVisibility: .visible) {144 Button("Delete all", role: .destructive) { clear() }145 Button("Cancel", role: .cancel) {}146 }147 }148149 // MARK: On device150151 private var localSection: some View {152 VStack(alignment: .leading, spacing: AstraSpace.md) {153 SectionTitle("On-device model", detail: "\(promptCount) fixed prompts")154 if let reason = app.local.mlxUnavailableReason {155 NoticeCard(.info, title: "No on-device benchmark here", message: reason + " No figure is shown rather than an invented one.")156 .accessibilityIdentifier("benchmark-unavailable")157 } else if app.local.installed.isEmpty {158 AstraCard {159 EmptyStateView(symbol: "arrow.down.circle", title: "No model installed", message: "Download an on-device model in Models to measure it on this iPhone.") {160 Button("Open Models") { app.tab = .models }.buttonStyle(AstraPrimaryButtonStyle())161 }162 .frame(maxWidth: .infinity)163 }164 } else {165 AstraCard {166 VStack(alignment: .leading, spacing: AstraSpace.md) {167 HStack {168 Text("Model").font(AstraFont.subheadline).foregroundStyle(AstraColor.textPrimary)169 Spacer(minLength: AstraSpace.sm)170 Picker("Model", selection: $localID) {171 ForEach(app.local.installed) { Text($0.card.displayName).tag(Optional($0.id)) }172 }173 .pickerStyle(.menu).tint(AstraColor.violet).disabled(isRunning)174 }175 .frame(minHeight: AstraSpace.hit)176 Toggle(isOn: $reload) {177 VStack(alignment: .leading, spacing: AstraSpace.xxs) {178 Text("Unload first").font(AstraFont.subheadline).foregroundStyle(AstraColor.textPrimary)179 Text("Gives a true cold start: the first prompt then pays for loading pages and compiling shaders.")180 .font(AstraFont.caption).foregroundStyle(AstraColor.textSecondary).fixedSize(horizontal: false, vertical: true)181 }182 }183 .tint(AstraColor.accentFill).disabled(isRunning)184 if isRunning {185 VStack(alignment: .leading, spacing: AstraSpace.sm) {186 AstraProgressBar(Double(progress) / Double(max(1, promptCount)))187 Text(progress == 0 ? "Loading the model and running the first prompt…" : "\(progress) of \(promptCount) prompts measured")188 .font(AstraFont.footnote).foregroundStyle(AstraColor.textSecondary)189 }190 Button { localTask?.cancel() } label: { Label("Stop", systemImage: "stop.fill") }191 .buttonStyle(AstraSecondaryButtonStyle()).accessibilityIdentifier("benchmark-stop")192 } else {193 Button { runLocal() } label: { Label("Run benchmark", systemImage: "play.fill") }194 .buttonStyle(AstraPrimaryButtonStyle()).disabled(localID == nil).accessibilityIdentifier("benchmark-run")195 }196 Text("Speed depends on heat, battery state and what else is running. Results describe this run on this device, not the model in general.")197 .font(AstraFont.caption).foregroundStyle(AstraColor.textTertiary).fixedSize(horizontal: false, vertical: true)198 }199 }200 if let localError { NoticeCard(.error, title: localError.title, message: localError.message) }201 if let result { resultCard(result) }202 }203 }204 }205206 private func resultCard(_ r: LocalBenchmarkResult) -> some View {207 AstraCard {208 VStack(alignment: .leading, spacing: AstraSpace.md) {209 SectionTitle("Result", detail: "Saved")210 VStack(alignment: .leading, spacing: AstraSpace.sm) {211 HStack(spacing: AstraSpace.sm) { AstraBadge("Cold", symbol: "snowflake", tone: .cyan); Text("First generation after loading").font(AstraFont.caption).foregroundStyle(AstraColor.textSecondary) }212 if let cold = r.cold {213 HStack(alignment: .top, spacing: AstraSpace.md) {214 MetricTile("First token", value: LabFormat.seconds(cold.timeToFirstToken), unit: "s", symbol: "bolt")215 MetricTile("Decode", value: LabFormat.rate(cold.decodeTokensPerSecond), unit: "tok/s", symbol: "speedometer")216 MetricTile("Load", value: LabFormat.seconds(r.loadTime), unit: "s", symbol: "arrow.down.to.line")217 }218 } else {219 Text("No cold sample: the model was already in memory. Turn on “Unload first” to measure one.")220 .font(AstraFont.footnote).foregroundStyle(AstraColor.textSecondary).fixedSize(horizontal: false, vertical: true)221 }222 }223 LabDivider()224 VStack(alignment: .leading, spacing: AstraSpace.sm) {225 HStack(spacing: AstraSpace.sm) { AstraBadge("Warm", symbol: "flame", tone: .warning); Text("Median of \(r.warm.count) later prompt\(r.warm.count == 1 ? "" : "s")").font(AstraFont.caption).foregroundStyle(AstraColor.textSecondary) }226 HStack(alignment: .top, spacing: AstraSpace.md) {227 MetricTile("First token", value: LabFormat.seconds(r.warmTimeToFirstToken), unit: "s", symbol: "bolt")228 MetricTile("Decode", value: LabFormat.rate(r.warmDecodeTokensPerSecond), unit: "tok/s", symbol: "speedometer")229 MetricTile("Peak memory", value: LabFormat.bytes(r.peakMemoryBytes), symbol: "memorychip")230 }231 }232 LabDivider()233 Text("Repeatability").font(AstraFont.captionEmphasis).foregroundStyle(AstraColor.textSecondary)234 VStack(alignment: .leading, spacing: 0) {235 FactRow(label: "Device", value: r.device.modelIdentifier, mono: true)236 FactRow(label: "System", value: "iOS \(r.device.osVersion)")237 FactRow(label: "Model revision", value: r.modelRevision.map { String($0.prefix(12)) }, mono: true)238 FactRow(label: "Thermal state", value: "\(r.thermalBefore.rawValue.capitalized) before, \(r.thermalAfter.rawValue.capitalized) after")239 FactRow(label: "Weights in memory", value: LabFormat.bytes(r.weightsMemoryBytes))240 FactRow(label: "Settings", value: "Temperature 0, seed 7, \(LocalBenchmark().maxOutputTokens) tokens max")241 }242 LabDivider()243 Text("Per prompt").font(AstraFont.captionEmphasis).foregroundStyle(AstraColor.textSecondary)244 ForEach(r.samples, id: \.promptID) { sample in245 VStack(alignment: .leading, spacing: AstraSpace.xxs) {246 HStack(spacing: AstraSpace.sm) {247 Text(sample.promptID).font(AstraFont.monoSmall).foregroundStyle(AstraColor.textPrimary)248 AstraBadge(sample.cold ? "Cold" : "Warm", tone: sample.cold ? .cyan : .warning)249 Spacer(minLength: 0)250 }251 Text([LabFormat.seconds(sample.timeToFirstToken).map { "first token \($0) s" }, LabFormat.rate(sample.decodeTokensPerSecond).map { "\($0) tok/s" },252 sample.outputTokens.map { "\($0) tokens out" }, sample.stopReason.label.lowercased()].compactMap { $0 }.joined(separator: " · "))253 .font(AstraFont.caption).foregroundStyle(AstraColor.textSecondary)254 Text(sample.outputPreview.isEmpty ? "No text was generated." : sample.outputPreview).font(AstraFont.caption).foregroundStyle(AstraColor.textTertiary).lineLimit(2)255 }256 .accessibilityElement(children: .combine)257 }258 }259 }260 .accessibilityIdentifier("benchmark-result")261 }262263 private func runLocal() {264 guard let id = localID, !isRunning else { return }265 let engine = app.local.engine, reload = reload266 isRunning = true; progress = 0; localError = nil; result = nil267 localTask = Task {268 let began = Date()269 let counter = Task { for await metrics in engine.monitor.updates() where metrics.modelID == id && metrics.startedAt >= began.addingTimeInterval(-1) { progress += 1 } }270 defer { counter.cancel(); isRunning = false; localTask = nil }271 do {272 let measured = try await LocalBenchmark().run(modelID: id, engine: engine, reload: reload)273 result = measured274 await save(measured)275 AstraHaptics.success()276 } catch {277 let mapped = AstraError.wrap(error)278 if mapped != .cancelled { localError = mapped; AstraHaptics.error() }279 }280 }281 }282283 private func save(_ r: LocalBenchmarkResult) async {284 let ref = ModelRef(provider: .mlx, modelID: r.modelID)285 var toSave: [BenchmarkRecord] = []286 if let cold = r.cold {287 toSave.append(BenchmarkRecord(model: ref, deviceModel: r.device.modelIdentifier, osVersion: r.device.osVersion, modelRevision: r.modelRevision, isColdStart: true,288 timeToFirstToken: cold.timeToFirstToken, tokensPerSecond: cold.decodeTokensPerSecond, promptTokens: cold.promptTokens,289 outputTokens: cold.outputTokens, peakMemoryBytes: cold.peakMemoryBytes.map { Int($0) }, thermalState: (cold.thermalAfter ?? r.thermalAfter).rawValue,290 createdAt: r.startedAt, notes: "Prompt “\(cold.promptID)”." + (LabFormat.seconds(r.loadTime).map { " Model load \($0) s." } ?? "")))291 }292 if !r.warm.isEmpty {293 toSave.append(BenchmarkRecord(model: ref, deviceModel: r.device.modelIdentifier, osVersion: r.device.osVersion, modelRevision: r.modelRevision, isColdStart: false,294 timeToFirstToken: r.warmTimeToFirstToken, tokensPerSecond: r.warmDecodeTokensPerSecond,295 peakMemoryBytes: r.peakMemoryBytes.map { Int($0) }, thermalState: r.thermalAfter.rawValue, createdAt: r.startedAt.addingTimeInterval(0.001),296 notes: "Median of \(r.warm.count) warm prompt\(r.warm.count == 1 ? "" : "s"). Thermal before: \(r.thermalBefore.rawValue)."))297 }298 for record in toSave {299 do { try await app.store.saveBenchmark(record) } catch { app.show(AstraError.wrap(error).message, .error) }300 }301 }302303 // MARK: Cloud probe304305 private var probeSection: some View {306 VStack(alignment: .leading, spacing: AstraSpace.md) {307 SectionTitle("Cloud latency probe", detail: "Spends a few tokens")308 AstraCard {309 VStack(alignment: .leading, spacing: AstraSpace.md) {310 ModelChoiceRow(ref: probeModel, placeholder: "Choose a cloud model") { showPicker = true }.accessibilityIdentifier("probe-model")311 if let probeModel, probeModel.provider.isLocal {312 Label("This is an on-device model: there is no network latency to probe. Use the benchmark above.", systemImage: "info.circle")313 .font(AstraFont.caption).foregroundStyle(AstraColor.textSecondary).fixedSize(horizontal: false, vertical: true)314 }315 Text("Sends one real request with a one-line prompt and an output limit of 16 tokens, on your own key. It costs a few tokens, shown below once the provider reports them.")316 .font(AstraFont.caption).foregroundStyle(AstraColor.textSecondary).fixedSize(horizontal: false, vertical: true)317 if let probe, probe.isRunning {318 Button { probe.stop() } label: { Label("Stop", systemImage: "stop.fill") }.buttonStyle(AstraSecondaryButtonStyle()).accessibilityIdentifier("probe-stop")319 } else {320 Button { runProbe() } label: { Label("Measure latency", systemImage: "antenna.radiowaves.left.and.right") }321 .buttonStyle(AstraPrimaryButtonStyle())322 .disabled(probeModel == nil || probeModel?.provider.isLocal == true)323 .accessibilityIdentifier("probe-run")324 }325 if let probe, probe.phase != .idle { LabRunResultView(run: probe, showText: false, showHeaders: true) }326 }327 }328 }329 }330331 private func runProbe() {332 guard let probeModel else { return }333 let controller = probe ?? LabRunController(store: app.store, library: app.library, orchestrator: app.orchestrator)334 probe = controller335 controller.run(model: probeModel, system: "", prompt: "Reply with the single word OK.", parameters: GenerationParameters(maxOutputTokens: 16), origin: .benchmark)336 }337338 private func saveProbe() {339 guard let probe, let model = probe.model, probe.phase == .done else { return }340 let device = app.local.device341 var notes = "Cloud latency probe."342 if let headers = LabFormat.seconds(probe.timeToHeaders) { notes += " Headers \(headers) s." }343 if let total = LabFormat.seconds(probe.timing?.totalTime) { notes += " Total \(total) s." }344 let record = BenchmarkRecord(model: model, deviceModel: device.modelIdentifier, osVersion: device.osVersion, isColdStart: false,345 timeToFirstToken: probe.timing?.timeToFirstToken, tokensPerSecond: nil, promptTokens: probe.usage?.inputTokens,346 outputTokens: probe.usage?.outputTokens, thermalState: ThermalLevel.current().rawValue, notes: notes)347 let store = app.store348 Task { try? await store.saveBenchmark(record) }349 }350351 // MARK: History352353 private struct Bar: Identifiable {354 var id: String355 var label: String356 var kind: String357 var value: Double358 }359360 private func bars(_ records: [BenchmarkRecord]) -> [Bar] {361 records.filter { $0.model.provider.isLocal && $0.tokensPerSecond != nil }.prefix(10).enumerated().map { index, record in362 let name = record.model.modelID.split(separator: "/").last.map(String.init) ?? record.model.modelID363 let short = name.count > 16 ? String(name.prefix(15)) + "…" : name364 return Bar(id: record.id, label: "\(index + 1). \(short)", kind: record.isColdStart ? "Cold" : "Warm", value: record.tokensPerSecond ?? 0)365 }366 }367368 @ViewBuilder private var historySection: some View {369 VStack(alignment: .leading, spacing: AstraSpace.md) {370 SectionTitle("Past measurements", detail: records.map { "\($0.count)" })371 if let historyError { NoticeCard(.error, title: "Records could not be read", message: historyError) }372 if let records {373 if records.isEmpty {374 AstraCard {375 Text("Nothing measured yet. Each run is kept here with the device, the system version, the model revision and the thermal state, so two results can be compared fairly.")376 .font(AstraFont.subheadline).foregroundStyle(AstraColor.textSecondary).fixedSize(horizontal: false, vertical: true)377 }378 } else {379 let bars = bars(records)380 if bars.count > 1 {381 AstraCard {382 VStack(alignment: .leading, spacing: AstraSpace.md) {383 SectionTitle("Decode speed", detail: "Tokens per second, newest first")384 Chart(bars) { bar in385 BarMark(x: .value("Tokens per second", bar.value), y: .value("Run", bar.label))386 .foregroundStyle(by: .value("Start", bar.kind)).cornerRadius(3)387 .annotation(position: .trailing, alignment: .leading, spacing: AstraSpace.xs) {388 Text(String(format: "%.1f", bar.value)).font(AstraFont.micro).foregroundStyle(AstraColor.textSecondary)389 }390 }391 .chartForegroundStyleScale(domain: ["Cold", "Warm"], range: [AstraColor.cyan, AstraColor.warning])392 .chartXAxis(.hidden)393 .chartXScale(domain: 0...max(bars.map(\.value).max() ?? 1, 1) * 1.25)394 .chartYAxis { AxisMarks(preset: .aligned, position: .leading) { _ in AxisValueLabel().font(AstraFont.caption).foregroundStyle(AstraColor.textPrimary) } }395 .frame(height: CGFloat(bars.count) * 34 + 40)396 .accessibilityLabel("Decode speed of past on-device runs")397 .accessibilityValue(bars.map { "\($0.label), \($0.kind.lowercased()): \(String(format: "%.1f", $0.value)) tokens per second" }.joined(separator: ". "))398 .accessibilityIdentifier("benchmark-chart")399 }400 }401 }402 ForEach(records) { record in recordCard(record) }403 }404 } else if historyError == nil { LabLoading() }405 }406 }407408 private func recordCard(_ record: BenchmarkRecord) -> some View {409 let isLocal = record.model.provider.isLocal410 return AstraCard {411 VStack(alignment: .leading, spacing: AstraSpace.sm) {412 HStack(alignment: .top, spacing: AstraSpace.sm) {413 ProviderLogo(ref: record.model, vendor: app.library.info(for: record.model)?.vendor, size: 30)414 VStack(alignment: .leading, spacing: AstraSpace.xs) {415 Text(app.library.info(for: record.model)?.displayName ?? record.model.modelID).font(AstraFont.bodyEmphasis).foregroundStyle(AstraColor.textPrimary).lineLimit(2)416 FlowLayout(spacing: AstraSpace.xs) {417 if isLocal { AstraBadge(record.isColdStart ? "Cold" : "Warm", symbol: record.isColdStart ? "snowflake" : "flame", tone: record.isColdStart ? .cyan : .warning) }418 else { AstraBadge("Cloud probe", symbol: "antenna.radiowaves.left.and.right") }419 AstraBadge(LabFormat.dateTime(record.createdAt), symbol: "calendar")420 }421 }422 Spacer(minLength: 0)423 IconButton("trash", label: "Delete this record", tint: AstraColor.textTertiary) { delete(record) }424 }425 HStack(alignment: .top, spacing: AstraSpace.md) {426 MetricTile("First token", value: LabFormat.seconds(record.timeToFirstToken), unit: "s", symbol: "bolt")427 if isLocal {428 MetricTile("Decode", value: LabFormat.rate(record.tokensPerSecond), unit: "tok/s", symbol: "speedometer")429 MetricTile("Peak memory", value: LabFormat.bytes(record.peakMemoryBytes.map { Int64($0) }), symbol: "memorychip")430 } else {431 MetricTile("Input", value: LabFormat.count(record.promptTokens), unit: "tok", symbol: "arrow.up")432 MetricTile("Output", value: LabFormat.count(record.outputTokens), unit: "tok", symbol: "arrow.down")433 }434 }435 Text([record.deviceModel, "iOS \(record.osVersion)", record.modelRevision.map { "rev \($0.prefix(7))" }, "thermal \(record.thermalState)"].compactMap { $0 }.joined(separator: " · "))436 .font(AstraFont.micro).foregroundStyle(AstraColor.textTertiary)437 if !record.notes.isEmpty { Text(record.notes).font(AstraFont.caption).foregroundStyle(AstraColor.textSecondary).fixedSize(horizontal: false, vertical: true) }438 }439 }440 }441442 private func loadHistory() async {443 do { records = try await app.store.benchmarks(); historyError = nil } catch { historyError = AstraError.wrap(error).message }444 }445446 private func delete(_ record: BenchmarkRecord) {447 let store = app.store448 Task { do { try await store.deleteBenchmark(id: record.id) } catch { app.show(AstraError.wrap(error).message, .error) } }449 }450451 private func clear() {452 let store = app.store453 Task {454 do { let n = try await store.deleteAllBenchmarks(); app.show("\(n) record\(n == 1 ? "" : "s") deleted", .success) }455 catch { app.show(AstraError.wrap(error).message, .error) }456 }457 }458459 private func export(_ format: ExportFormat) {460 let store = app.store461 Task {462 do { exported = app.exportedFile(try await store.exportBenchmarks(format), name: "astra-benchmarks", ext: format.fileExtension) }463 catch { app.show(AstraError.wrap(error).message, .error) }464 }465 }466}467