SPB Git

Repository Public MIT

BenchmarksLab.swift 27.3 KB 467 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Packages/AstraUI/Sources/Lab/BenchmarksLab.swift10// Module           : Lab11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import SwiftUI89import Charts90import AstraCore91import AstraDesign92import AstraLocal93import AstraPersistence9495/// Benchmarks: a fixed prompt set on an installed on-device model (cold and warm kept apart), and a tiny latency probe for cloud models.96struct BenchmarksLabView: View {97    @Environment(AppModel.self) private var app98    @State private var localID: String?99    @State private var reload = true100    @State private var isRunning = false101    @State private var progress = 0102    @State private var result: LocalBenchmarkResult?103    @State private var localError: AstraError?104    @State private var localTask: Task<Void, Never>?105106    @State private var probeModel: ModelRef?107    @State private var probe: LabRunController?108    @State private var showPicker = false109110    @State private var records: [BenchmarkRecord]?111    @State private var historyError: String?112    @State private var exported: ExportedFile?113    @State private var confirmClear = false114115    private let promptCount = LocalBenchmark.standardPrompts.count116117    var body: some View {118        LabPage(title: "Benchmarks") {119            localSection120            probeSection121            historySection122        }123        .toolbar {124            ToolbarItem(placement: .topBarTrailing) {125                Menu {126                    Button { export(.csv) } label: { Label("Export CSV", systemImage: "tablecells") }127                    Button { export(.json) } label: { Label("Export JSON", systemImage: "curlybraces") }128                    Button { export(.markdown) } label: { Label("Export Markdown", systemImage: "doc.plaintext") }129                    Divider()130                    Button(role: .destructive) { confirmClear = true } label: { Label("Delete all records", systemImage: "trash") }131                } label: { Label("More", systemImage: "ellipsis.circle") }132                .disabled(records?.isEmpty ?? true)133                .accessibilityIdentifier("benchmarks-menu")134            }135        }136        .task {137            if localID == nil { localID = app.local.installed.first?.id }138            for await _ in app.store.changes(in: [.benchmark]) { await loadHistory() }139        }140        .sheet(isPresented: $showPicker) { ModelPickerSheet(selected: probeModel, title: "Model to probe") { probeModel = $0 } }141        .sheet(item: $exported) { ShareSheet(items: [$0.url]) }142        .onChange(of: probe?.phase) { _, phase in if phase == .done { saveProbe() } }143        .confirmationDialog("Delete every benchmark record?", isPresented: $confirmClear, titleVisibility: .visible) {144            Button("Delete all", role: .destructive) { clear() }145            Button("Cancel", role: .cancel) {}146        }147    }148149    // MARK: On device150151    private var localSection: some View {152        VStack(alignment: .leading, spacing: AstraSpace.md) {153            SectionTitle("On-device model", detail: "\(promptCount) fixed prompts")154            if let reason = app.local.mlxUnavailableReason {155                NoticeCard(.info, title: "No on-device benchmark here", message: reason + " No figure is shown rather than an invented one.")156                    .accessibilityIdentifier("benchmark-unavailable")157            } else if app.local.installed.isEmpty {158                AstraCard {159                    EmptyStateView(symbol: "arrow.down.circle", title: "No model installed", message: "Download an on-device model in Models to measure it on this iPhone.") {160                        Button("Open Models") { app.tab = .models }.buttonStyle(AstraPrimaryButtonStyle())161                    }162                    .frame(maxWidth: .infinity)163                }164            } else {165                AstraCard {166                    VStack(alignment: .leading, spacing: AstraSpace.md) {167                        HStack {168                            Text("Model").font(AstraFont.subheadline).foregroundStyle(AstraColor.textPrimary)169                            Spacer(minLength: AstraSpace.sm)170                            Picker("Model", selection: $localID) {171                                ForEach(app.local.installed) { Text($0.card.displayName).tag(Optional($0.id)) }172                            }173                            .pickerStyle(.menu).tint(AstraColor.violet).disabled(isRunning)174                        }175                        .frame(minHeight: AstraSpace.hit)176                        Toggle(isOn: $reload) {177                            VStack(alignment: .leading, spacing: AstraSpace.xxs) {178                                Text("Unload first").font(AstraFont.subheadline).foregroundStyle(AstraColor.textPrimary)179                                Text("Gives a true cold start: the first prompt then pays for loading pages and compiling shaders.")180                                    .font(AstraFont.caption).foregroundStyle(AstraColor.textSecondary).fixedSize(horizontal: false, vertical: true)181                            }182                        }183                        .tint(AstraColor.accentFill).disabled(isRunning)184                        if isRunning {185                            VStack(alignment: .leading, spacing: AstraSpace.sm) {186                                AstraProgressBar(Double(progress) / Double(max(1, promptCount)))187                                Text(progress == 0 ? "Loading the model and running the first prompt…" : "\(progress) of \(promptCount) prompts measured")188                                    .font(AstraFont.footnote).foregroundStyle(AstraColor.textSecondary)189                            }190                            Button { localTask?.cancel() } label: { Label("Stop", systemImage: "stop.fill") }191                                .buttonStyle(AstraSecondaryButtonStyle()).accessibilityIdentifier("benchmark-stop")192                        } else {193                            Button { runLocal() } label: { Label("Run benchmark", systemImage: "play.fill") }194                                .buttonStyle(AstraPrimaryButtonStyle()).disabled(localID == nil).accessibilityIdentifier("benchmark-run")195                        }196                        Text("Speed depends on heat, battery state and what else is running. Results describe this run on this device, not the model in general.")197                            .font(AstraFont.caption).foregroundStyle(AstraColor.textTertiary).fixedSize(horizontal: false, vertical: true)198                    }199                }200                if let localError { NoticeCard(.error, title: localError.title, message: localError.message) }201                if let result { resultCard(result) }202            }203        }204    }205206    private func resultCard(_ r: LocalBenchmarkResult) -> some View {207        AstraCard {208            VStack(alignment: .leading, spacing: AstraSpace.md) {209                SectionTitle("Result", detail: "Saved")210                VStack(alignment: .leading, spacing: AstraSpace.sm) {211                    HStack(spacing: AstraSpace.sm) { AstraBadge("Cold", symbol: "snowflake", tone: .cyan); Text("First generation after loading").font(AstraFont.caption).foregroundStyle(AstraColor.textSecondary) }212                    if let cold = r.cold {213                        HStack(alignment: .top, spacing: AstraSpace.md) {214                            MetricTile("First token", value: LabFormat.seconds(cold.timeToFirstToken), unit: "s", symbol: "bolt")215                            MetricTile("Decode", value: LabFormat.rate(cold.decodeTokensPerSecond), unit: "tok/s", symbol: "speedometer")216                            MetricTile("Load", value: LabFormat.seconds(r.loadTime), unit: "s", symbol: "arrow.down.to.line")217                        }218                    } else {219                        Text("No cold sample: the model was already in memory. Turn on “Unload first” to measure one.")220                            .font(AstraFont.footnote).foregroundStyle(AstraColor.textSecondary).fixedSize(horizontal: false, vertical: true)221                    }222                }223                LabDivider()224                VStack(alignment: .leading, spacing: AstraSpace.sm) {225                    HStack(spacing: AstraSpace.sm) { AstraBadge("Warm", symbol: "flame", tone: .warning); Text("Median of \(r.warm.count) later prompt\(r.warm.count == 1 ? "" : "s")").font(AstraFont.caption).foregroundStyle(AstraColor.textSecondary) }226                    HStack(alignment: .top, spacing: AstraSpace.md) {227                        MetricTile("First token", value: LabFormat.seconds(r.warmTimeToFirstToken), unit: "s", symbol: "bolt")228                        MetricTile("Decode", value: LabFormat.rate(r.warmDecodeTokensPerSecond), unit: "tok/s", symbol: "speedometer")229                        MetricTile("Peak memory", value: LabFormat.bytes(r.peakMemoryBytes), symbol: "memorychip")230                    }231                }232                LabDivider()233                Text("Repeatability").font(AstraFont.captionEmphasis).foregroundStyle(AstraColor.textSecondary)234                VStack(alignment: .leading, spacing: 0) {235                    FactRow(label: "Device", value: r.device.modelIdentifier, mono: true)236                    FactRow(label: "System", value: "iOS \(r.device.osVersion)")237                    FactRow(label: "Model revision", value: r.modelRevision.map { String($0.prefix(12)) }, mono: true)238                    FactRow(label: "Thermal state", value: "\(r.thermalBefore.rawValue.capitalized) before, \(r.thermalAfter.rawValue.capitalized) after")239                    FactRow(label: "Weights in memory", value: LabFormat.bytes(r.weightsMemoryBytes))240                    FactRow(label: "Settings", value: "Temperature 0, seed 7, \(LocalBenchmark().maxOutputTokens) tokens max")241                }242                LabDivider()243                Text("Per prompt").font(AstraFont.captionEmphasis).foregroundStyle(AstraColor.textSecondary)244                ForEach(r.samples, id: \.promptID) { sample in245                    VStack(alignment: .leading, spacing: AstraSpace.xxs) {246                        HStack(spacing: AstraSpace.sm) {247                            Text(sample.promptID).font(AstraFont.monoSmall).foregroundStyle(AstraColor.textPrimary)248                            AstraBadge(sample.cold ? "Cold" : "Warm", tone: sample.cold ? .cyan : .warning)249                            Spacer(minLength: 0)250                        }251                        Text([LabFormat.seconds(sample.timeToFirstToken).map { "first token \($0) s" }, LabFormat.rate(sample.decodeTokensPerSecond).map { "\($0) tok/s" },252                              sample.outputTokens.map { "\($0) tokens out" }, sample.stopReason.label.lowercased()].compactMap { $0 }.joined(separator: " · "))253                            .font(AstraFont.caption).foregroundStyle(AstraColor.textSecondary)254                        Text(sample.outputPreview.isEmpty ? "No text was generated." : sample.outputPreview).font(AstraFont.caption).foregroundStyle(AstraColor.textTertiary).lineLimit(2)255                    }256                    .accessibilityElement(children: .combine)257                }258            }259        }260        .accessibilityIdentifier("benchmark-result")261    }262263    private func runLocal() {264        guard let id = localID, !isRunning else { return }265        let engine = app.local.engine, reload = reload266        isRunning = true; progress = 0; localError = nil; result = nil267        localTask = Task {268            let began = Date()269            let counter = Task { for await metrics in engine.monitor.updates() where metrics.modelID == id && metrics.startedAt >= began.addingTimeInterval(-1) { progress += 1 } }270            defer { counter.cancel(); isRunning = false; localTask = nil }271            do {272                let measured = try await LocalBenchmark().run(modelID: id, engine: engine, reload: reload)273                result = measured274                await save(measured)275                AstraHaptics.success()276            } catch {277                let mapped = AstraError.wrap(error)278                if mapped != .cancelled { localError = mapped; AstraHaptics.error() }279            }280        }281    }282283    private func save(_ r: LocalBenchmarkResult) async {284        let ref = ModelRef(provider: .mlx, modelID: r.modelID)285        var toSave: [BenchmarkRecord] = []286        if let cold = r.cold {287            toSave.append(BenchmarkRecord(model: ref, deviceModel: r.device.modelIdentifier, osVersion: r.device.osVersion, modelRevision: r.modelRevision, isColdStart: true,288                                          timeToFirstToken: cold.timeToFirstToken, tokensPerSecond: cold.decodeTokensPerSecond, promptTokens: cold.promptTokens,289                                          outputTokens: cold.outputTokens, peakMemoryBytes: cold.peakMemoryBytes.map { Int($0) }, thermalState: (cold.thermalAfter ?? r.thermalAfter).rawValue,290                                          createdAt: r.startedAt, notes: "Prompt “\(cold.promptID)”." + (LabFormat.seconds(r.loadTime).map { " Model load \($0) s." } ?? "")))291        }292        if !r.warm.isEmpty {293            toSave.append(BenchmarkRecord(model: ref, deviceModel: r.device.modelIdentifier, osVersion: r.device.osVersion, modelRevision: r.modelRevision, isColdStart: false,294                                          timeToFirstToken: r.warmTimeToFirstToken, tokensPerSecond: r.warmDecodeTokensPerSecond,295                                          peakMemoryBytes: r.peakMemoryBytes.map { Int($0) }, thermalState: r.thermalAfter.rawValue, createdAt: r.startedAt.addingTimeInterval(0.001),296                                          notes: "Median of \(r.warm.count) warm prompt\(r.warm.count == 1 ? "" : "s"). Thermal before: \(r.thermalBefore.rawValue)."))297        }298        for record in toSave {299            do { try await app.store.saveBenchmark(record) } catch { app.show(AstraError.wrap(error).message, .error) }300        }301    }302303    // MARK: Cloud probe304305    private var probeSection: some View {306        VStack(alignment: .leading, spacing: AstraSpace.md) {307            SectionTitle("Cloud latency probe", detail: "Spends a few tokens")308            AstraCard {309                VStack(alignment: .leading, spacing: AstraSpace.md) {310                    ModelChoiceRow(ref: probeModel, placeholder: "Choose a cloud model") { showPicker = true }.accessibilityIdentifier("probe-model")311                    if let probeModel, probeModel.provider.isLocal {312                        Label("This is an on-device model: there is no network latency to probe. Use the benchmark above.", systemImage: "info.circle")313                            .font(AstraFont.caption).foregroundStyle(AstraColor.textSecondary).fixedSize(horizontal: false, vertical: true)314                    }315                    Text("Sends one real request with a one-line prompt and an output limit of 16 tokens, on your own key. It costs a few tokens, shown below once the provider reports them.")316                        .font(AstraFont.caption).foregroundStyle(AstraColor.textSecondary).fixedSize(horizontal: false, vertical: true)317                    if let probe, probe.isRunning {318                        Button { probe.stop() } label: { Label("Stop", systemImage: "stop.fill") }.buttonStyle(AstraSecondaryButtonStyle()).accessibilityIdentifier("probe-stop")319                    } else {320                        Button { runProbe() } label: { Label("Measure latency", systemImage: "antenna.radiowaves.left.and.right") }321                            .buttonStyle(AstraPrimaryButtonStyle())322                            .disabled(probeModel == nil || probeModel?.provider.isLocal == true)323                            .accessibilityIdentifier("probe-run")324                    }325                    if let probe, probe.phase != .idle { LabRunResultView(run: probe, showText: false, showHeaders: true) }326                }327            }328        }329    }330331    private func runProbe() {332        guard let probeModel else { return }333        let controller = probe ?? LabRunController(store: app.store, library: app.library, orchestrator: app.orchestrator)334        probe = controller335        controller.run(model: probeModel, system: "", prompt: "Reply with the single word OK.", parameters: GenerationParameters(maxOutputTokens: 16), origin: .benchmark)336    }337338    private func saveProbe() {339        guard let probe, let model = probe.model, probe.phase == .done else { return }340        let device = app.local.device341        var notes = "Cloud latency probe."342        if let headers = LabFormat.seconds(probe.timeToHeaders) { notes += " Headers \(headers) s." }343        if let total = LabFormat.seconds(probe.timing?.totalTime) { notes += " Total \(total) s." }344        let record = BenchmarkRecord(model: model, deviceModel: device.modelIdentifier, osVersion: device.osVersion, isColdStart: false,345                                     timeToFirstToken: probe.timing?.timeToFirstToken, tokensPerSecond: nil, promptTokens: probe.usage?.inputTokens,346                                     outputTokens: probe.usage?.outputTokens, thermalState: ThermalLevel.current().rawValue, notes: notes)347        let store = app.store348        Task { try? await store.saveBenchmark(record) }349    }350351    // MARK: History352353    private struct Bar: Identifiable {354        var id: String355        var label: String356        var kind: String357        var value: Double358    }359360    private func bars(_ records: [BenchmarkRecord]) -> [Bar] {361        records.filter { $0.model.provider.isLocal && $0.tokensPerSecond != nil }.prefix(10).enumerated().map { index, record in362            let name = record.model.modelID.split(separator: "/").last.map(String.init) ?? record.model.modelID363            let short = name.count > 16 ? String(name.prefix(15)) + "…" : name364            return Bar(id: record.id, label: "\(index + 1). \(short)", kind: record.isColdStart ? "Cold" : "Warm", value: record.tokensPerSecond ?? 0)365        }366    }367368    @ViewBuilder private var historySection: some View {369        VStack(alignment: .leading, spacing: AstraSpace.md) {370            SectionTitle("Past measurements", detail: records.map { "\($0.count)" })371            if let historyError { NoticeCard(.error, title: "Records could not be read", message: historyError) }372            if let records {373                if records.isEmpty {374                    AstraCard {375                        Text("Nothing measured yet. Each run is kept here with the device, the system version, the model revision and the thermal state, so two results can be compared fairly.")376                            .font(AstraFont.subheadline).foregroundStyle(AstraColor.textSecondary).fixedSize(horizontal: false, vertical: true)377                    }378                } else {379                    let bars = bars(records)380                    if bars.count > 1 {381                        AstraCard {382                            VStack(alignment: .leading, spacing: AstraSpace.md) {383                                SectionTitle("Decode speed", detail: "Tokens per second, newest first")384                                Chart(bars) { bar in385                                    BarMark(x: .value("Tokens per second", bar.value), y: .value("Run", bar.label))386                                        .foregroundStyle(by: .value("Start", bar.kind)).cornerRadius(3)387                                        .annotation(position: .trailing, alignment: .leading, spacing: AstraSpace.xs) {388                                            Text(String(format: "%.1f", bar.value)).font(AstraFont.micro).foregroundStyle(AstraColor.textSecondary)389                                        }390                                }391                                .chartForegroundStyleScale(domain: ["Cold", "Warm"], range: [AstraColor.cyan, AstraColor.warning])392                                .chartXAxis(.hidden)393                                .chartXScale(domain: 0...max(bars.map(\.value).max() ?? 1, 1) * 1.25)394                                .chartYAxis { AxisMarks(preset: .aligned, position: .leading) { _ in AxisValueLabel().font(AstraFont.caption).foregroundStyle(AstraColor.textPrimary) } }395                                .frame(height: CGFloat(bars.count) * 34 + 40)396                                .accessibilityLabel("Decode speed of past on-device runs")397                                .accessibilityValue(bars.map { "\($0.label), \($0.kind.lowercased()): \(String(format: "%.1f", $0.value)) tokens per second" }.joined(separator: ". "))398                                .accessibilityIdentifier("benchmark-chart")399                            }400                        }401                    }402                    ForEach(records) { record in recordCard(record) }403                }404            } else if historyError == nil { LabLoading() }405        }406    }407408    private func recordCard(_ record: BenchmarkRecord) -> some View {409        let isLocal = record.model.provider.isLocal410        return AstraCard {411            VStack(alignment: .leading, spacing: AstraSpace.sm) {412                HStack(alignment: .top, spacing: AstraSpace.sm) {413                    ProviderLogo(ref: record.model, vendor: app.library.info(for: record.model)?.vendor, size: 30)414                    VStack(alignment: .leading, spacing: AstraSpace.xs) {415                        Text(app.library.info(for: record.model)?.displayName ?? record.model.modelID).font(AstraFont.bodyEmphasis).foregroundStyle(AstraColor.textPrimary).lineLimit(2)416                        FlowLayout(spacing: AstraSpace.xs) {417                            if isLocal { AstraBadge(record.isColdStart ? "Cold" : "Warm", symbol: record.isColdStart ? "snowflake" : "flame", tone: record.isColdStart ? .cyan : .warning) }418                            else { AstraBadge("Cloud probe", symbol: "antenna.radiowaves.left.and.right") }419                            AstraBadge(LabFormat.dateTime(record.createdAt), symbol: "calendar")420                        }421                    }422                    Spacer(minLength: 0)423                    IconButton("trash", label: "Delete this record", tint: AstraColor.textTertiary) { delete(record) }424                }425                HStack(alignment: .top, spacing: AstraSpace.md) {426                    MetricTile("First token", value: LabFormat.seconds(record.timeToFirstToken), unit: "s", symbol: "bolt")427                    if isLocal {428                        MetricTile("Decode", value: LabFormat.rate(record.tokensPerSecond), unit: "tok/s", symbol: "speedometer")429                        MetricTile("Peak memory", value: LabFormat.bytes(record.peakMemoryBytes.map { Int64($0) }), symbol: "memorychip")430                    } else {431                        MetricTile("Input", value: LabFormat.count(record.promptTokens), unit: "tok", symbol: "arrow.up")432                        MetricTile("Output", value: LabFormat.count(record.outputTokens), unit: "tok", symbol: "arrow.down")433                    }434                }435                Text([record.deviceModel, "iOS \(record.osVersion)", record.modelRevision.map { "rev \($0.prefix(7))" }, "thermal \(record.thermalState)"].compactMap { $0 }.joined(separator: " · "))436                    .font(AstraFont.micro).foregroundStyle(AstraColor.textTertiary)437                if !record.notes.isEmpty { Text(record.notes).font(AstraFont.caption).foregroundStyle(AstraColor.textSecondary).fixedSize(horizontal: false, vertical: true) }438            }439        }440    }441442    private func loadHistory() async {443        do { records = try await app.store.benchmarks(); historyError = nil } catch { historyError = AstraError.wrap(error).message }444    }445446    private func delete(_ record: BenchmarkRecord) {447        let store = app.store448        Task { do { try await store.deleteBenchmark(id: record.id) } catch { app.show(AstraError.wrap(error).message, .error) } }449    }450451    private func clear() {452        let store = app.store453        Task {454            do { let n = try await store.deleteAllBenchmarks(); app.show("\(n) record\(n == 1 ? "" : "s") deleted", .success) }455            catch { app.show(AstraError.wrap(error).message, .error) }456        }457    }458459    private func export(_ format: ExportFormat) {460        let store = app.store461        Task {462            do { exported = app.exportedFile(try await store.exportBenchmarks(format), name: "astra-benchmarks", ext: format.fileExtension) }463            catch { app.show(AstraError.wrap(error).message, .error) }464        }465    }466}467
Files main
Menu

Theme