Repository Public License
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Audio9// File : Core/Sources/orix-bench/main.swift10// Description : Command-line benchmarks: DSP throughput, on-device11// transcription error rates, Foundation Models notes. Runs12// on macOS.13// Module : orix-bench14// Repository : https://git.spboucher.ai/orix-audio15// Created : 2026-11-2016// Last Modified : 2026-11-2017//18// =============================================================================19// AUTHOR & PROFESSIONAL AFFILIATION20// =============================================================================21//22// Author : Simon-Pierre Boucher23// Position : Professor of Finance24// Institution : Université du Québec en Outaouais (UQO)25// Department : Département des sciences administratives26// Organization : SP Boucher27//28// Website : https://spboucher.ai29// Git : https://git.spboucher.ai30//31// =============================================================================32// OFFICIAL CONTACT DIRECTORY33// =============================================================================34//35// General : info@spboucher.ai36// Professional : contact@spboucher.ai37// Direct : simon@spboucher.ai38// Research : research@spboucher.ai39// Development : dev@spboucher.ai40// Support : support@spboucher.ai41// Security/Admin : admin@spboucher.ai42// Notifications : notifications@spboucher.ai43//44// Preferred Contact: support@spboucher.ai45//46// =============================================================================47// COPYRIGHT & INTELLECTUAL PROPERTY48// =============================================================================49//50// Copyright (c) 202651// Simon-Pierre Boucher52//53// All rights reserved where applicable.54//55// This file is governed by the applicable project license,56// copyright notices, institutional agreements and relevant law.57//58// Unless expressly authorized by the applicable license or59// rights holder, copying, redistribution, sublicensing,60// publication and commercial exploitation are not permitted.61//62// Existing third-party rights and notices remain unaffected.63//64// =============================================================================65// CONFIDENTIALITY & SECURITY66// =============================================================================67//68// Classification : Public69// Distribution : Public repository; use is governed by the project license70// Security Contact : admin@spboucher.ai71//72// Access to this source does not independently grant permission73// to redistribute or disclose confidential materials.74//75// Do not embed credentials, private keys, access tokens,76// certificates or confidential personal information.77//78// =============================================================================79// LICENSE & MAINTENANCE80// =============================================================================81//82// License : Proprietary, all rights reserved (see license file)83// License File : LICENSE84// Maintainer : Simon-Pierre Boucher85// Technical Contact: dev@spboucher.ai86//87// =============================================================================88// END OF SP BOUCHER SOURCE HEADER89// =============================================================================9091import AVFoundation92import Foundation93import ORIXAudioCore9495func measure(_ repeats: Int = 1, _ body: () throws -> Void) rethrows -> Double {96 let start = DispatchTime.now().uptimeNanoseconds97 for _ in 0..<repeats { try body() }98 return Double(DispatchTime.now().uptimeNanoseconds - start) / 1e9 / Double(repeats)99}100101func row(_ name: String, _ value: String) { print("| \(name) | \(value) |") }102103func dspBenchmarks() throws {104 print("\n## DSP throughput (single thread)\n\n| Operation | Result |\n|---|---|")105 let fs = 48_000.0106 for n in [1024, 4096, 16384] {107 let fft = RealFFT(size: n, window: .hann)108 let x = SignalGenerator.pinkNoise(rms: 0.1, count: n, seed: 1)109 var out = [Float](repeating: 0, count: n / 2 + 1)110 let t = measure(2000) { x.withUnsafeBufferPointer { p in out.withUnsafeMutableBufferPointer { fft.squaredMagnitudes(p.baseAddress!, into: $0.baseAddress!); fft.scale($0.baseAddress!, to: .amplitude, sampleRate: fs) } } }111 row("FFT \(n) (window + transform + scaling)", String(format: "%.1f µs", t * 1e6))112 }113 let minute = SignalGenerator.pinkNoise(rms: 0.1, count: Int(fs) * 60, seed: 2)114 var an = SpectrumAnalyzer(sampleRate: fs)115 let ta = measure(2000) { minute.withUnsafeBufferPointer { _ = an.process($0.baseAddress!, elapsed: 0.03) } }116 row("Live spectrum frame (4096-point, smoothing, peak hold, bands)", String(format: "%.1f µs", ta * 1e6))117 an = SpectrumAnalyzer(sampleRate: fs)118 let tw = measure { _ = WaveformProcessor.summarize(minute, sampleRate: fs) }119 row("Waveform summary with band colours, 60 s of audio", String(format: "%.0f ms (%.0f× real time)", tw * 1e3, 60 / tw))120 var tile: SpectrogramTile!121 let ts = measure { tile = SpectrogramProcessor.compute(minute, sampleRate: fs) }122 row("Spectrogram 2048/512, 60 s of audio (\(tile.columnCount) columns)", String(format: "%.0f ms (%.0f× real time)", ts * 1e3, 60 / ts))123 let slice = SpectrogramTile(sampleRate: fs, configuration: tile.configuration, firstColumn: 0, columnCount: 512, binCount: tile.binCount, values: Array(tile.values[0..<(512 * tile.binCount)]))124 let tr = measure(20) { _ = SpectrogramRasterizer.rgba(tile: slice, rows: 512, axis: .logarithmic, minFrequency: 20, maxFrequency: 20_000, floorDB: -100, ceilingDB: 0, palette: .orixSpectrum) }125 row("Rasterise a 512 × 512 spectrogram tile (CPU, palette LUT)", String(format: "%.2f ms", tr * 1e3))126 var meter = LoudnessMeter(sampleRate: fs, channels: 1)127 let tl = measure { meter.push(mono: minute); _ = meter.finish() }128 row("BS.1770 loudness + true peak, 60 s mono", String(format: "%.0f ms (%.0f× real time)", tl * 1e3, 60 / tl))129130 // Whole-file analysis and export on a 10-minute stereo file.131 let long = SignalGenerator.pinkNoise(rms: 0.1, count: Int(fs) * 600, seed: 3)132 let url = FileManager.default.temporaryDirectory.appendingPathComponent("orix-bench.caf")133 let w = try AudioFileWriter(url: url, format: .caf24, inputSampleRate: fs, channels: 2)134 for start in stride(from: 0, to: long.count, by: 480_000) { let c = Array(long[start..<min(long.count, start + 480_000)]); try w.write([c, c]) }135 try w.finish()136 let tAnalyze = try measure { _ = try AudioAnalyzer.analyze(url: url) }137 row("Full analysis pass, 10 min stereo 48 kHz file", String(format: "%.2f s (%.0f× real time)", tAnalyze, 600 / tAnalyze))138 let doc = EditDocument(sourceName: "a", duration: 600)139 for format in [AudioExportFormat.wav16, .alac, .aac] {140 let out = FileManager.default.temporaryDirectory.appendingPathComponent("orix-bench-out.\(format.fileExtension)")141 let t = try measure { try EditRenderer(document: doc, sourceURLs: [url]).export(to: out, format: format) }142 let size = (try FileManager.default.attributesOfItem(atPath: out.path)[.size] as? NSNumber)?.doubleValue ?? 0143 row("Export 10 min stereo → \(format.displayName)", String(format: "%.2f s (%.0f× real time), %.1f MB", t, 600 / t, size / 1e6))144 try? FileManager.default.removeItem(at: out)145 }146 var nr = doc; nr.setEffects([.noiseReduction(strength: 0.6), .voiceEnhance])147 let out = FileManager.default.temporaryDirectory.appendingPathComponent("orix-bench-nr.wav")148 let tnr = try measure { try EditRenderer(document: nr, sourceURLs: [url]).export(to: out, format: .wav16) }149 row("Noise reduction + voice enhance + export, 10 min stereo", String(format: "%.2f s (%.0f× real time)", tnr, 600 / tnr))150 try? FileManager.default.removeItem(at: out); try? FileManager.default.removeItem(at: url)151}152153/// `orix-bench transcribe <audio> <locale> <reference.txt>`: prints WER / CER as JSON.154func transcriptionBenchmark(_ args: [String]) async throws {155 guard args.count >= 3 else { print("usage: orix-bench transcribe <audio> <locale> <reference.txt>"); return }156 let url = URL(fileURLWithPath: args[0]), locale = Locale(identifier: args[1])157 let reference = try String(contentsOfFile: args[2], encoding: .utf8)158 func trace(_ m: String) { FileHandle.standardError.write(Data("[bench] \(m)\n".utf8)) }159 trace("resolving recogniser for \(args[1])")160 let path = await TranscriptionEngine.path(for: locale)161 trace("path: \(path)")162 let start = Date()163 let transcript = try await TranscriptionEngine.transcribe(url: url, locale: locale, allowModelDownload: true)164 let elapsed = Date().timeIntervalSince(start)165 let duration = try AudioFileReader(url: url).duration166 let wer = TranscriptionScoring.wordErrorRate(reference: reference, hypothesis: transcript.plainText)167 let cer = TranscriptionScoring.characterErrorRate(reference: reference, hypothesis: transcript.plainText)168 let result: [String: Any] = [169 "file": url.lastPathComponent, "locale": args[1], "path": "\(path)", "engine": transcript.engine, "audioSeconds": duration, "elapsedSeconds": elapsed,170 "realTimeFactor": duration / max(elapsed, 1e-6), "segments": transcript.segments.count, "referenceWords": wer.referenceLength,171 "wer": wer.rate, "substitutions": wer.substitutions, "deletions": wer.deletions, "insertions": wer.insertions, "cer": cer.rate,172 "timedWords": transcript.segments.reduce(0) { $0 + $1.words.count }, "hypothesis": transcript.plainText,173 ]174 print(String(decoding: try JSONSerialization.data(withJSONObject: result, options: [.prettyPrinted, .sortedKeys]), as: UTF8.self))175 if args.count > 3 { try TranscriptExport.json(transcript).write(to: URL(fileURLWithPath: args[3])) }176}177178/// `orix-bench notes <transcript.json> [question]`: runs Foundation Models notes (and an optional question) on a saved transcript.179func notesBenchmark(_ args: [String]) async throws {180 print("availability: \(MeetingIntelligence.availability)")181 guard let first = args.first else { return }182 let d = JSONDecoder(); d.dateDecodingStrategy = .iso8601183 let transcript = try d.decode(Transcript.self, from: Data(contentsOf: URL(fileURLWithPath: first)))184 let start = Date()185 let notes = try await MeetingIntelligence.notes(for: transcript, style: .meeting)186 print(String(format: "generated in %.1f s", Date().timeIntervalSince(start)))187 print(NotesExport.markdown(notes, transcript: transcript))188 if let out = ProcessInfo.processInfo.environment["ORIX_NOTES_OUT"] {189 let e = JSONEncoder(); e.dateEncodingStrategy = .iso8601190 try e.encode(notes).write(to: URL(fileURLWithPath: out))191 }192 let all = notes.keyPoints + notes.actionItems + notes.decisions193 print("grounded items: \(all.filter(\.isGrounded).count) / \(all.count)")194 for q in args.dropFirst() {195 let a = try await MeetingIntelligence.answer(question: q, transcript: transcript)196 print("\nQ: \(q)\nA (\(a.isAnswerable ? "answerable" : "not answerable"), \(a.segmentIDs.count) citations): \(a.text)")197 }198}199200let arguments = Array(CommandLine.arguments.dropFirst())201do {202 switch arguments.first {203 case "transcribe": try await transcriptionBenchmark(Array(arguments.dropFirst()))204 case "notes": try await notesBenchmark(Array(arguments.dropFirst()))205 case "install":206 let locale = Locale(identifier: arguments.count > 1 ? arguments[1] : "en-US")207 print("path before:", await TranscriptionEngine.path(for: locale))208 try await TranscriptionEngine.installModel(for: locale) { p in FileHandle.standardError.write(Data(String(format: "[bench] download %.0f %%\n", p * 100).utf8)) }209 print("path after:", await TranscriptionEngine.path(for: locale))210 case "locales":211 let locales = await TranscriptionEngine.supportedLocales()212 print(locales.map { $0.identifier(.bcp47) }.joined(separator: " "))213 default: try dspBenchmarks()214 }215} catch {216 print("error: \(error.localizedDescription)")217 exit(1)218}219