SPB Git

Repository Public MIT

SimulationTests.swift 17.3 KB 248 lines swift

1// ORIX Calc2// File: Packages/ORIXStatistics/Tests/ORIXStatisticsTests/SimulationTests.swift3// Description: Monte Carlo runner (reproducibility, cancellation, progress), scenario tables, sampling distributions and bootstrap intervals.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Testing9import ORIXNumerics10@testable import ORIXStatistics1112/// Collects progress callbacks from a @Sendable closure.13private final class ProgressLog: @unchecked Sendable {14    private let lock = NSLock()15    private var items: [MonteCarloProgress] = []16    func add(_ p: MonteCarloProgress) { lock.lock(); items.append(p); lock.unlock() }17    var all: [MonteCarloProgress] { lock.lock(); defer { lock.unlock() }; return items }18}1920@Suite struct MonteCarloTests {21    @Test func estimatesKnownQuantitiesWithinTheirStandardError() throws {22        // π/4 as the probability that a uniform point of the unit square falls inside the quarter disc.23        let pi = try MonteCarlo.run(trials: 200_000, seed: 314, event: { $0 == 1 }) { g in24            let x = Sampling.uniform(using: &g), y = Sampling.uniform(using: &g)25            return x * x + y * y <= 1 ? 1 : 026        }27        let p = try #require(pi.eventProbability), se = try #require(pi.eventStandardError)28        #expect(abs(p - Double.pi / 4) < 4 * se && se < 0.001)29        #expect(pi.mean == p && pi.trials == 200_000 && pi.seed == 314)30        // Sum of two dice: mean 7, variance 35/6, P(sum = 7) = 1/6, median 7.31        let die = try DiscreteUniformDistribution(lowerBound: 1, upperBound: 6)32        let dice = try MonteCarlo.run(trials: 120_000, seed: 7, quantiles: [0.25, 0.5, 0.75], binning: .fixedWidth(1), event: { $0 == 7 }) { g in33            die.sample(using: &g) + die.sample(using: &g)34        }35        #expect(abs(dice.mean - 7) < 4 * dice.standardError)36        #expect(close(dice.standardDeviation, (35.0 / 6).squareRoot(), rtol: 0.01))37        #expect(abs(try #require(dice.eventProbability) - 1.0 / 6) < 4 * (try #require(dice.eventStandardError)))38        #expect(dice.quantiles.map(\.value) == [5, 7, 9] && dice.minimum == 2 && dice.maximum == 12)39        #expect(dice.histogram.counts.count == 11 && dice.histogram.total == 120_000)40        #expect(dice.meanLower < 7 && dice.meanUpper > 7)41        #expect(close(dice.probability(where: { $0 >= 10 }), 6.0 / 36, rtol: 0.03))42        #expect(try dice.quantile(0.5) == 7)43        #expect(dice.verification.isVerified)44    }4546    @Test func sameSeedSameResultDifferentSeedDifferentResult() throws {47        let normal = try NormalDistribution(mean: 100, standardDeviation: 15)48        func run(_ seed: UInt64) throws -> MonteCarloSummary { try MonteCarlo.run(trials: 5000, seed: seed) { g in normal.sample(using: &g) } }49        let a = try run(2026), b = try run(2026), c = try run(2027)50        #expect(a == b)                                   // every statistic and every outcome, bit for bit51        #expect(a.values != c.values && a.mean != c.mean)52        #expect(try MonteCarlo.summarize(a.values, seed: 2026) == a)53    }5455    @Test func asyncRunMatchesSyncAndReportsProgress() async throws {56        let log = ProgressLog()57        let exponential = try ExponentialDistribution(rate: 0.5)58        let sync = try MonteCarlo.run(trials: 10_000, seed: 99, event: { $0 > 4 }) { g in exponential.sample(using: &g) }59        let asyncResult = try await MonteCarlo.runAsync(trials: 10_000, seed: 99, progressInterval: 2500, event: { $0 > 4 }, progress: { log.add($0) }) { g in60            exponential.sample(using: &g)61        }62        #expect(asyncResult == sync)63        #expect(log.all.map(\.completed) == [2500, 5000, 7500, 10_000] && log.all.last?.fraction == 1)64        #expect(abs(try #require(sync.eventProbability) - exp(-2)) < 4 * (try #require(sync.eventStandardError)))65        var calls = 066        _ = try MonteCarlo.run(trials: 1050, seed: 1, progressInterval: 500, progress: { _ in calls += 1 }) { g in Sampling.uniform(using: &g) }67        #expect(calls == 3)                               // 500, 1000 and the final 105068    }6970    @Test func asyncRunIsCancellable() async throws {71        let task = Task {72            try await MonteCarlo.runAsync(trials: 40_000_000, seed: 5, progressInterval: 1000) { g in Sampling.standardNormal(using: &g) }73        }74        try await Task.sleep(nanoseconds: 30_000_000)75        task.cancel()76        let outcome = await task.result77        switch outcome {78        case .success: Issue.record("a cancelled simulation must not return a summary")79        case .failure(let error): #expect(error is CancellationError)80        }81        // Already-cancelled tasks stop before the first trial.82        let early = Task { () -> MonteCarloSummary in83            withUnsafeCurrentTask { $0?.cancel() }84            return try await MonteCarlo.runAsync(trials: 10, seed: 1) { g in Sampling.uniform(using: &g) }85        }86        #expect(await { if case .failure(let e) = await early.result { return e is CancellationError }; return false }())87    }8889    @Test func errorsAndWarnings() throws {90        #expect(throws: StatisticsError.self) { try MonteCarlo.run(trials: 0, seed: 1) { _ in 1 } }91        #expect(throws: StatisticsError.probabilityOutOfRange(2)) { try MonteCarlo.run(trials: 10, seed: 1, quantiles: [2]) { _ in 1 } }92        #expect(throws: StatisticsError.nonFiniteValue("The outcome of trial 3")) {93            var k = 094            return try MonteCarlo.run(trials: 10, seed: 1) { _ in k += 1; return k == 3 ? .nan : 1 }95        }96        #expect(throws: StatisticsError.emptyData) { try MonteCarlo.summarize([]) }97        // A trial closure may throw its own errors; they propagate.98        struct Stop: Error {}99        #expect(throws: Stop.self) { try MonteCarlo.run(trials: 10, seed: 1) { _ in throw Stop() } }100        let few = try MonteCarlo.run(trials: 50, seed: 3, event: { $0 > 0.99 }) { g in Sampling.uniform(using: &g) }101        #expect(few.verification.warnings.count == 2)     // fewer than 1000 trials, and a rare event seen too rarely102        let one = try MonteCarlo.run(trials: 1, seed: 3) { _ in 4.5 }103        #expect(one.standardDeviation == 0 && one.mean == 4.5 && one.histogram.total == 1)104    }105106    @Test func samplingDistributionOfTheMean() throws {107        // Means of n = 25 from an exponential(1): centre 1, standard error 1/√25 = 0.2 (central limit theorem).108        let d = try ExponentialDistribution(rate: 1)109        let s = try MonteCarlo.samplingDistribution(of: { try Descriptive.mean($0) }, sampleSize: 25, replications: 20_000, seed: 11, from: d)110        #expect(abs(s.mean - 1) < 4 * s.standardError && close(s.standardDeviation, 0.2, rtol: 0.03))111        #expect(s.quantiles.map(\.probability) == [0.025, 0.25, 0.5, 0.75, 0.975])112        let population = [2.0, 4, 4, 4, 5, 5, 7, 9]      // σ = 2, μ = 5113        let f = try MonteCarlo.samplingDistribution(of: { try Descriptive.mean($0) }, sampleSize: 16, replications: 20_000, seed: 12, fromPopulation: population)114        #expect(abs(f.mean - 5) < 4 * f.standardError && close(f.standardDeviation, 0.5, rtol: 0.03))115        #expect(throws: StatisticsError.self) { try MonteCarlo.samplingDistribution(of: { $0[0] }, sampleSize: 0, replications: 10, seed: 1, from: d) }116        #expect(throws: StatisticsError.emptyData) { try MonteCarlo.samplingDistribution(of: { $0[0] }, sampleSize: 2, replications: 10, seed: 1, fromPopulation: []) }117    }118}119120@Suite struct ScenarioTests {121    static func loan(_ inputs: [String: Double]) throws -> [String: Double] {122        let principal = inputs["principal"]!, rate = inputs["rate"]! / 12, n = inputs["years"]! * 12123        let payment = principal * rate / (1 - pow(1 + rate, -n))124        return ["payment": payment, "totalInterest": payment * n - principal]125    }126127    @Test func scenarioTable() throws {128        let table = try ScenarioAnalysis.evaluate([129            Scenario(name: "Base", inputs: ["principal": 300_000, "rate": 0.05, "years": 25]),130            Scenario(name: "Rates up", inputs: ["principal": 300_000, "rate": 0.07, "years": 25]),131            Scenario(name: "Shorter", inputs: ["principal": 300_000, "rate": 0.05, "years": 15]),132        ], model: Self.loan)133        #expect(table.inputNames == ["principal", "rate", "years"] && table.outputNames == ["payment", "totalInterest"])134        #expect(close(try #require(table.output("payment", in: "Base")), 1753.7701246, rtol: 1e-9))135        #expect(try #require(table.output("payment", in: "Rates up")) > 1753.78 && table.output("nothing", in: "Base") == nil)136        #expect(table.column("payment")?.count == 3 && table.changesFromBase[0].outputs == [0, 0])137        #expect(close(table.changesFromBase[1].inputs[1], 0.02, rtol: 1e-12) && table.changesFromBase[2].outputs[1] < 0)138        let ordered = try ScenarioAnalysis.evaluate([Scenario(name: "Base", inputs: ["principal": 1000, "rate": 0.05, "years": 1])], outputs: ["totalInterest"], model: Self.loan)139        #expect(ordered.outputNames == ["totalInterest"])140    }141142    @Test func oneWaySensitivity() throws {143        let base = ["principal": 300_000.0, "rate": 0.05, "years": 25]144        let t = try ScenarioAnalysis.sensitivity(base: base, varying: "rate", values: [0.03, 0.05, 0.07], model: Self.loan)145        let payments = try #require(t.column("payment"))146        #expect(payments == payments.sorted() && t.rows.count == 3 && t.rows[1].inputs == [300_000, 0.05, 25])147        #expect(throws: StatisticsError.self) { try ScenarioAnalysis.sensitivity(base: base, varying: "tax", values: [1], model: Self.loan) }148        #expect(throws: StatisticsError.emptyData) { try ScenarioAnalysis.sensitivity(base: base, varying: "rate", values: [], model: Self.loan) }149    }150151    @Test func scenarioErrors() {152        #expect(throws: StatisticsError.emptyData) { try ScenarioAnalysis.evaluate([], model: Self.loan) }153        #expect(throws: StatisticsError.self) {154            try ScenarioAnalysis.evaluate([Scenario(name: "A", inputs: ["x": 1]), Scenario(name: "A", inputs: ["x": 2])]) { ["y": $0["x"]!] }155        }156        #expect(throws: StatisticsError.self) {157            try ScenarioAnalysis.evaluate([Scenario(name: "A", inputs: ["x": 1]), Scenario(name: "B", inputs: ["z": 2])]) { _ in ["y": 1] }158        }159        #expect(throws: StatisticsError.self) { try ScenarioAnalysis.evaluate([Scenario(name: "A", inputs: ["x": 0])]) { ["y": 1 / $0["x"]!] } }160        #expect(throws: StatisticsError.self) { try ScenarioAnalysis.evaluate([Scenario(name: "A", inputs: ["x": 0])], outputs: ["missing"]) { _ in ["y": 1] } }161        #expect(throws: StatisticsError.self) { try ScenarioAnalysis.evaluate([Scenario(name: "A", inputs: ["x": .nan])]) { _ in ["y": 1] } }162    }163}164165@Suite struct BootstrapTests {166    @Test func intervalArithmeticMatchesTheReferenceOnFixedReplicates() throws {167        // Same replicates as the Python reference, so the three interval formulas are compared exactly (no random stream involved).168        let cases = try Reference.cases("bootstrap")169        #expect(cases.count == 6)170        for c in cases {171            let reps = c.doubles("replicates"), theta = c.double("estimate"), level = c.double("level")172            let pct = try Bootstrap.interval(replicates: reps, estimate: theta, level: level, method: .percentile)173            #expect(close(pct.lower, c.doubles("percentile")[0], rtol: 1e-12) && close(pct.upper, c.doubles("percentile")[1], rtol: 1e-12))174            let basic = try Bootstrap.interval(replicates: reps, estimate: theta, level: level, method: .basic)175            #expect(close(basic.lower, c.doubles("basic")[0], rtol: 1e-12) && close(basic.upper, c.doubles("basic")[1], rtol: 1e-12))176            let bca = try Bootstrap.interval(replicates: reps, estimate: theta, jackknife: c.doubles("jackknife"), level: level, method: .bca)177            #expect(close(try #require(bca.biasCorrection), c.double("z0"), rtol: 1e-10, atol: 1e-12), "\(c.string("name")) z0")178            #expect(close(try #require(bca.acceleration), c.double("acceleration"), rtol: 1e-10, atol: 1e-14), "\(c.string("name")) a")179            #expect(close(bca.lower, c.doubles("bca")[0], rtol: 1e-9) && close(bca.upper, c.doubles("bca")[1], rtol: 1e-9), "\(c.string("name")) BCa")180        }181    }182183    @Test func endToEndAgreesWithScipyBootstrap() throws {184        // Different random streams, so agreement is statistical: limits within 6% of the interval width of scipy's 20 000-resample result.185        for c in try Reference.cases("bootstrap") where c.has("scipy_bca") {186            let data = c.doubles("data"), name = c.string("name")187            let statistic: ([Double]) throws -> Double = switch name {188            case "lognormalMean": { try Descriptive.mean($0) }189            case "normalSD": { try Descriptive.standardDeviation($0) }190            default: { try Descriptive.median($0) }191            }192            for (method, key) in [(BootstrapMethod.percentile, "scipy_percentile"), (.basic, "scipy_basic"), (.bca, "scipy_bca")] {193                let r = try Bootstrap.confidenceInterval(data: data, level: 0.95, resamples: 20_000, seed: 2026, method: method, statistic: statistic)194                let e = c.doubles(key), width = e[1] - e[0]195                #expect(abs(r.lower - e[0]) < 0.06 * width && abs(r.upper - e[1]) < 0.06 * width, "\(name) \(method): (\(r.lower), \(r.upper)) vs \(e)")196                #expect(close(r.standardError, c.double("scipy_se"), rtol: 0.04), "\(name) se \(r.standardError) vs \(c.double("scipy_se"))")197                #expect(r.estimate == c.double("estimate") || close(r.estimate, c.double("estimate"), rtol: 1e-13))198                #expect(r.lower < r.upper && r.replicates.count == 20_000)199            }200        }201    }202203    @Test func seedReproducibilityAndDefaults() throws {204        let data = [12.1, 9.8, 15.3, 11.0, 10.4, 13.7, 9.1, 14.9, 12.5, 11.8, 16.2, 10.9, 13.1, 12.0, 11.4, 9.9, 14.2]205        let a = try Bootstrap.confidenceInterval(data: data, seed: 5) { try Descriptive.mean($0) }206        let b = try Bootstrap.confidenceInterval(data: data, seed: 5) { try Descriptive.mean($0) }207        let c = try Bootstrap.confidenceInterval(data: data, seed: 6) { try Descriptive.mean($0) }208        #expect(a == b && a.replicates != c.replicates)209        #expect(a.method == .bca && a.replicates.count == 2000 && a.level == 0.95 && a.seed == 5)       // documented defaults210        #expect(a.biasCorrection != nil && a.acceleration != nil && a.verification.isVerified)211        #expect(a.lower < a.estimate && a.estimate < a.upper)212        // For the mean, the bootstrap standard error estimates s·√((n−1)/n)/√n and the interval resembles the t interval.213        let n = Double(data.count)214        #expect(close(a.standardError, try Descriptive.standardDeviation(data, sample: false) / n.squareRoot(), rtol: 0.05))215        let t = try ConfidenceIntervals.meanT(data: data)216        #expect(abs(a.lower - t.lower) < 0.25 * t.marginOfError && abs(a.upper - t.upper) < 0.25 * t.marginOfError)217        #expect(abs(a.bias) < 3 * a.standardError / (2000.0).squareRoot() + 1e-12)218        #expect(a.explanation.plainText.contains("Bootstrap confidence interval (BCa)") && a.explanation.inputs.contains { $0.label == "seed" && $0.value == "5" })219    }220221    @Test func pairedDataThroughTheIndexForm() throws {222        // Correlation needs rows resampled together: the index-based entry point does that.223        let x = [1.0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]224        let y = [2.1, 2.9, 4.2, 3.8, 6.1, 5.2, 8.3, 7.1, 9.9, 9.2, 12.4, 10.8, 13.9, 13.1, 16.5, 15.2]225        let r = try Bootstrap.confidenceInterval(count: x.count, resamples: 3000, seed: 77, method: .percentile) { idx in226            try Descriptive.correlation(idx.map { x[$0] }, idx.map { y[$0] })227        }228        #expect(close(r.estimate, try Descriptive.correlation(x, y), rtol: 1e-14) && r.lower > 0.9 && r.upper <= 1 && r.lower < r.estimate)229    }230231    @Test func errorPaths() {232        let data = [1.0, 2, 3, 4, 5, 6, 7, 8, 9, 10]233        #expect(throws: StatisticsError.emptyData) { try Bootstrap.confidenceInterval(data: [], seed: 1) { $0[0] } }234        #expect(throws: StatisticsError.insufficientSample(required: 2, actual: 1)) { try Bootstrap.confidenceInterval(data: [4], seed: 1) { $0[0] } }235        #expect(throws: StatisticsError.self) { try Bootstrap.confidenceInterval(data: data, resamples: 10, seed: 1) { $0[0] } }236        #expect(throws: StatisticsError.probabilityOutOfRange(1)) { try Bootstrap.confidenceInterval(data: data, level: 1, seed: 1) { $0[0] } }237        // A statistic that is not finite on some resample (variance-free resample → correlation undefined) is reported, not hidden.238        #expect(throws: StatisticsError.self) { try Bootstrap.confidenceInterval(data: [1, 1, 1, 2], resamples: 500, seed: 1) { try Descriptive.zScores($0)[0] } }239        // BCa without jackknife values, and BCa when every replicate is on one side of the estimate.240        #expect(throws: StatisticsError.self) { try Bootstrap.interval(replicates: data, estimate: 5, method: .bca) }241        #expect(throws: StatisticsError.self) { try Bootstrap.interval(replicates: data, estimate: 50, jackknife: data, method: .bca) }242        #expect(throws: StatisticsError.self) { try Bootstrap.interval(replicates: [1], estimate: 1, method: .percentile) }243        // A constant statistic has no BCa interval (all replicates tie with the estimate → share = ½, a = 0 → degenerate but defined).244        let constant = try? Bootstrap.confidenceInterval(data: data, resamples: 200, seed: 1, method: .percentile) { _ in 3 }245        #expect(constant?.lower == 3 && constant?.upper == 3 && constant?.verification.status == .warning)246    }247}248
Files main
Menu

Theme