SPB Git

Repository Public

AccuracyTests.swift 15.4 KB 299 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX DJ9// File             : Packages/ORIXDJCore/Tests/ORIXAnalysisTests/AccuracyTests.swift10// Description      : Measured accuracy of tempo, beat phase, downbeat and key11//                    estimation on synthesized audio; prints the release-report12//                    table.13// Module           : ORIXAnalysisTests14// Repository       : https://git.spboucher.ai/orix-dj15// Created          : 2026-11-2116// Last Modified    : 2026-11-2117//18// =============================================================================19// AUTHOR & PROFESSIONAL AFFILIATION20// =============================================================================21//22// Author           : Simon-Pierre Boucher23// Position         : Professor of Finance24// Institution      : Université du Québec en Outaouais (UQO)25// Department       : Département des sciences administratives26// Organization     : SP Boucher27//28// Website          : https://spboucher.ai29// Git              : https://git.spboucher.ai30//31// =============================================================================32// OFFICIAL CONTACT DIRECTORY33// =============================================================================34//35// General          : info@spboucher.ai36// Professional     : contact@spboucher.ai37// Direct           : simon@spboucher.ai38// Research         : research@spboucher.ai39// Development      : dev@spboucher.ai40// Support          : support@spboucher.ai41// Security/Admin   : admin@spboucher.ai42// Notifications    : notifications@spboucher.ai43//44// Preferred Contact: support@spboucher.ai45//46// =============================================================================47// COPYRIGHT & INTELLECTUAL PROPERTY48// =============================================================================49//50// Copyright (c) 202651// Simon-Pierre Boucher52//53// All rights reserved where applicable.54//55// This file is governed by the applicable project license,56// copyright notices, institutional agreements and relevant law.57//58// Unless expressly authorized by the applicable license or59// rights holder, copying, redistribution, sublicensing,60// publication and commercial exploitation are not permitted.61//62// Existing third-party rights and notices remain unaffected.63//64// =============================================================================65// CONFIDENTIALITY & SECURITY66// =============================================================================67//68// Classification   : Public69// Distribution     : Public repository; use is governed by the project license70// Security Contact : admin@spboucher.ai71//72// Access to this source does not independently grant permission73// to redistribute or disclose confidential materials.74//75// Do not embed credentials, private keys, access tokens,76// certificates or confidential personal information.77//78// =============================================================================79// LICENSE & MAINTENANCE80// =============================================================================81//82// License          : None specified (no license file in this repository)83// Maintainer       : Simon-Pierre Boucher84// Technical Contact: dev@spboucher.ai85//86// =============================================================================87// END OF SP BOUCHER SOURCE HEADER88// =============================================================================8990import Foundation91import Testing92import ORIXAnalysis9394let testSampleRate = 44100.09596func analyze(_ left: [Float], _ right: [Float]? = nil, sampleRate: Double = testSampleRate,97             options: AnalysisOptions = .init()) throws -> AnalysisResult {98    try TrackAnalyzer.analyze(left: left, right: right ?? left, sampleRate: sampleRate, options: options)99}100101enum TempoOutcome: Sendable { case correct, double, half, other, none }102103func classify(estimated: Double?, truth: Double) -> TempoOutcome {104    guard let estimated else { return .none }105    let ratio = estimated / truth106    if abs(ratio - 1) < 0.01 { return .correct }107    if abs(ratio - 2) < 0.02 { return .double }108    if abs(ratio - 0.5) < 0.005 { return .half }109    return .other110}111112/// Signed circular difference a - b modulo `period`, in -period/2 ..< period/2.113func circularDifference(_ a: Double, _ b: Double, period: Double) -> Double {114    var d = (a - b).truncatingRemainder(dividingBy: period)115    if d >= period / 2 { d -= period }116    if d < -period / 2 { d += period }117    return d118}119120struct TempoRow: Sendable {121    var group: String122    var truth: Double123    var estimated: Double?124    var outcome: TempoOutcome125    var phaseErrorMs: Double?126    var downbeatCorrect: Bool?127    var bpmConfidence: Double128    var stability: Double129    var downbeatConfidence: Double130}131132struct KeyRow: Sendable {133    var group: String134    var truth: KeyEstimate135    var estimated: KeyEstimate?136    var relation: String137}138139func keyRelation(estimated: KeyEstimate?, truth: KeyEstimate) -> String {140    guard let e = estimated else { return "none" }141    if e.tonic == truth.tonic && e.mode == truth.mode { return "exact" }142    if e.mode != truth.mode && e.camelotNumber == truth.camelotNumber { return "relative" }143    if e.mode == truth.mode {144        let d = ((e.tonic - truth.tonic) % 12 + 12) % 12145        if d == 7 || d == 5 { return "fifth" }146    }147    if e.tonic == truth.tonic { return "parallel" }148    return "other"149}150151func measureTempo(group: String, bpm: Double, offset: Double, left: [Float], right: [Float]?, hasDownbeat: Bool) -> TempoRow {152    let result = try? analyze(left, right)153    let grid = result?.grid154    let outcome = classify(estimated: grid?.bpm, truth: bpm)155    var phase: Double?156    var downbeat: Bool?157    if let grid, outcome == .correct || outcome == .double {158        let beat = 60 / grid.bpm159        // Compare at the middle of the track so a small tempo error is charged to the phase as it would be heard.160        let middle = offset + ((Double(left.count) / testSampleRate / 2 - offset) / (60 / bpm)).rounded() * (60 / bpm)161        phase = circularDifference(grid.nearestBeatSeconds(to: middle), middle, period: beat) * 1000162        if hasDownbeat && outcome == .correct {163            let bar = 4 * 60 / bpm164            let estimatedBar = grid.firstBeatSeconds + Double(grid.downbeatIndex) * beat165            downbeat = abs(circularDifference(estimatedBar, offset, period: bar)) < 60 / bpm / 2166        }167    }168    return TempoRow(group: group, truth: bpm, estimated: grid?.bpm, outcome: outcome, phaseErrorMs: phase, downbeatCorrect: downbeat,169                    bpmConfidence: result?.bpmConfidence ?? 0, stability: result?.tempoStability ?? 0,170                    downbeatConfidence: result?.downbeatConfidence ?? 0)171}172173let tempoTable: [Double] = [70, 87.5, 100, 120, 124, 128, 133.3, 140, 150, 174]174175func pad(_ s: String, _ width: Int) -> String { s.count >= width ? s : s + String(repeating: " ", count: width - s.count) }176func fmt(_ v: Double, _ digits: Int = 3) -> String { String(format: "%.\(digits)f", v) }177178@Suite struct AccuracyTests {179    @Test func accuracyReport() async throws {180        let seconds = 40.0181        // Tempo / phase / downbeat rows.182        let tempoRows: [TempoRow] = await withTaskGroup(of: TempoRow.self) { group in183            for (i, bpm) in tempoTable.enumerated() {184                let offset = 0.031 + 0.0173 * Double(i)185                group.addTask {186                    let x = SignalSynth.clickTrack(bpm: bpm, seconds: seconds, sampleRate: testSampleRate, offsetSeconds: offset)187                    return measureTempo(group: "click", bpm: bpm, offset: offset, left: x, right: nil, hasDownbeat: false)188                }189                for (s, style) in SignalSynth.TrackSpec.Style.allCases.enumerated() {190                    group.addTask {191                        let spec = SignalSynth.TrackSpec(bpm: bpm, tonic: (i * 5 + s * 3) % 12, mode: (i + s) % 2 == 0 ? .minor : .major,192                                                         seconds: seconds, style: style, seed: UInt64(100 + i * 7 + s),193                                                         offsetSeconds: offset)194                        let t = SignalSynth.track(spec, sampleRate: testSampleRate)195                        return measureTempo(group: style.rawValue, bpm: bpm, offset: offset, left: t.left, right: t.right, hasDownbeat: true)196                    }197                }198            }199            var rows: [TempoRow] = []200            for await row in group { rows.append(row) }201            return rows202        }203        // Key rows: 24 keys x 3 styles.204        let keyStyles: [(SignalSynth.TrackSpec.Style, Double)] = [(.house, 124), (.breakbeat, 133.3), (.ambient, 100)]205        let keyRows: [KeyRow] = await withTaskGroup(of: KeyRow.self) { group in206            for (s, entry) in keyStyles.enumerated() {207                for tonic in 0..<12 {208                    for mode in [KeyMode.major, KeyMode.minor] {209                        group.addTask {210                            let spec = SignalSynth.TrackSpec(bpm: entry.1, tonic: tonic, mode: mode, seconds: seconds, style: entry.0,211                                                             seed: UInt64(500 + tonic + s * 31), offsetSeconds: 0.02)212                            let t = SignalSynth.track(spec, sampleRate: testSampleRate)213                            let truth = KeyEstimate(tonic: tonic, mode: mode, confidence: 1)214                            let estimated = (try? analyze(t.left, t.right))?.key215                            return KeyRow(group: entry.0.rawValue, truth: truth, estimated: estimated,216                                          relation: keyRelation(estimated: estimated, truth: truth))217                        }218                    }219                }220            }221            var rows: [KeyRow] = []222            for await row in group { rows.append(row) }223            return rows224        }225226        var lines: [String] = []227        lines.append("=== ORIX-ANALYSIS-ACCURACY ===")228        lines.append("Synthetic audio only (SignalSynth), \(Int(seconds)) s per track, 44.1 kHz, bpmRange 70...180.")229        lines.append("Tempi: " + tempoTable.map { fmt($0, 1) }.joined(separator: ", "))230        lines.append("")231        lines.append("TEMPO / BEAT PHASE / DOWNBEAT  (BPM error over tempo-correct tracks = within 1 %; phase over correct + double-time)")232        lines.append(pad("group", 11) + pad("n", 4) + pad("correct", 9) + pad("double", 8) + pad("half", 6) + pad("other", 7) + pad("nogrid", 8)233            + pad("bpmErr mean", 13) + pad("bpmErr max", 12) + pad("phase mean ms", 15) + pad("phase max ms", 14) + pad("downbeat", 10)234            + pad("conf", 6) + pad("stab", 6) + "dbConf")235        let groups = ["click"] + SignalSynth.TrackSpec.Style.allCases.map(\.rawValue)236        func summarise(_ name: String, _ rows: [TempoRow]) -> String {237            func count(_ o: TempoOutcome) -> Int { rows.filter { $0.outcome == o }.count }238            let correct = rows.filter { $0.outcome == .correct }239            let errors = correct.map { abs($0.estimated! - $0.truth) }240            let phases = rows.compactMap { $0.phaseErrorMs }.map { abs($0) }241            let downbeats = rows.compactMap { $0.downbeatCorrect }242            func mean(_ a: [Double]) -> String { a.isEmpty ? "-" : fmt(a.reduce(0, +) / Double(a.count)) }243            func maxOf(_ a: [Double]) -> String { a.isEmpty ? "-" : fmt(a.max()!) }244            let n = Double(max(1, rows.count))245            return pad(name, 11) + pad("\(rows.count)", 4) + pad("\(count(.correct))", 9) + pad("\(count(.double))", 8) + pad("\(count(.half))", 6)246                + pad("\(count(.other))", 7) + pad("\(count(.none))", 8) + pad(mean(errors), 13) + pad(maxOf(errors), 12)247                + pad(mean(phases), 15) + pad(maxOf(phases), 14)248                + pad(downbeats.isEmpty ? "-" : "\(downbeats.filter { $0 }.count)/\(downbeats.count)", 10)249                + pad(fmt(rows.map(\.bpmConfidence).reduce(0, +) / n, 2), 6) + pad(fmt(rows.map(\.stability).reduce(0, +) / n, 2), 6)250                + fmt(rows.map(\.downbeatConfidence).reduce(0, +) / n, 2)251        }252        for g in groups { lines.append(summarise(g, tempoRows.filter { $0.group == g })) }253        let tracks = tempoRows.filter { $0.group != "click" }254        lines.append(summarise("all tracks", tracks))255        let octave = tracks.filter { $0.outcome == .double || $0.outcome == .half }.count256        lines.append("Octave error rate on tracks: \(octave)/\(tracks.count) = \(fmt(100 * Double(octave) / Double(tracks.count), 1)) %")257        lines.append("")258        lines.append("Per-track detail (estimated BPM; * = not within 1 % of the truth)")259        lines.append(pad("bpm", 8) + groups.map { pad($0, 12) }.joined())260        for bpm in tempoTable {261            var line = pad(fmt(bpm, 1), 8)262            for g in groups {263                let row = tempoRows.first { $0.group == g && $0.truth == bpm }264                let text = row?.estimated.map { fmt($0, 2) + (row?.outcome == .correct ? "" : "*") } ?? "nil*"265                line += pad(text, 12)266            }267            lines.append(line)268        }269        lines.append("")270        lines.append("KEY  (24 keys per style)")271        lines.append(pad("style", 11) + pad("exact", 7) + pad("relative", 10) + pad("fifth", 7) + pad("parallel", 10) + pad("other", 7) + pad("none", 6) + "mean conf")272        func keyLine(_ name: String, _ rows: [KeyRow]) -> String {273            func count(_ r: String) -> String { "\(rows.filter { $0.relation == r }.count)" }274            let confidence = rows.compactMap { $0.estimated?.confidence }275            return pad(name, 11) + pad(count("exact"), 7) + pad(count("relative"), 10) + pad(count("fifth"), 7) + pad(count("parallel"), 10)276                + pad(count("other"), 7) + pad(count("none"), 6) + fmt(confidence.reduce(0, +) / Double(max(1, confidence.count)), 2)277        }278        for entry in keyStyles { lines.append(keyLine(entry.0.rawValue, keyRows.filter { $0.group == entry.0.rawValue })) }279        lines.append(keyLine("all", keyRows))280        let wrongKeys = keyRows.filter { $0.relation != "exact" }281            .map { "\($0.group) \($0.truth.shortName)->\($0.estimated?.shortName ?? "nil")" }.sorted()282        if !wrongKeys.isEmpty { lines.append("Key errors: " + wrongKeys.joined(separator: ", ")) }283        lines.append("=== END ===")284        print(lines.joined(separator: "\n"))285286        // Regression floors (see the printed table for the measured values).287        let clicks = tempoRows.filter { $0.group == "click" }288        #expect(clicks.allSatisfy { $0.outcome == .correct && abs($0.estimated! - $0.truth) < 0.02 && abs($0.phaseErrorMs ?? 99) < 2 })289        #expect(tracks.filter { $0.outcome == .correct }.count >= 32)290        #expect(tracks.allSatisfy { $0.outcome != .other && $0.outcome != .none })291        #expect(tracks.filter { $0.outcome == .correct }.allSatisfy { abs($0.estimated! - $0.truth) < 0.03 })292        let drumPhases = tracks.filter { $0.group != "ambient" }.compactMap { $0.phaseErrorMs }.map { abs($0) }293        #expect(drumPhases.reduce(0, +) / Double(max(1, drumPhases.count)) < 3)294        let downbeats = tracks.compactMap { $0.downbeatCorrect }295        #expect(downbeats.filter { $0 }.count * 10 >= downbeats.count * 8)296        #expect(keyRows.filter { $0.relation == "exact" }.count >= 64)297    }298}299
Files main
Menu

Theme