Repository Public MIT
1// ORIX Calc2// File: Packages/ORIXStatistics/Tests/ORIXStatisticsTests/SimulationTests.swift3// Description: Monte Carlo runner (reproducibility, cancellation, progress), scenario tables, sampling distributions and bootstrap intervals.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Testing9import ORIXNumerics10@testable import ORIXStatistics1112/// Collects progress callbacks from a @Sendable closure.13private final class ProgressLog: @unchecked Sendable {14 private let lock = NSLock()15 private var items: [MonteCarloProgress] = []16 func add(_ p: MonteCarloProgress) { lock.lock(); items.append(p); lock.unlock() }17 var all: [MonteCarloProgress] { lock.lock(); defer { lock.unlock() }; return items }18}1920@Suite struct MonteCarloTests {21 @Test func estimatesKnownQuantitiesWithinTheirStandardError() throws {22 // π/4 as the probability that a uniform point of the unit square falls inside the quarter disc.23 let pi = try MonteCarlo.run(trials: 200_000, seed: 314, event: { $0 == 1 }) { g in24 let x = Sampling.uniform(using: &g), y = Sampling.uniform(using: &g)25 return x * x + y * y <= 1 ? 1 : 026 }27 let p = try #require(pi.eventProbability), se = try #require(pi.eventStandardError)28 #expect(abs(p - Double.pi / 4) < 4 * se && se < 0.001)29 #expect(pi.mean == p && pi.trials == 200_000 && pi.seed == 314)30 // Sum of two dice: mean 7, variance 35/6, P(sum = 7) = 1/6, median 7.31 let die = try DiscreteUniformDistribution(lowerBound: 1, upperBound: 6)32 let dice = try MonteCarlo.run(trials: 120_000, seed: 7, quantiles: [0.25, 0.5, 0.75], binning: .fixedWidth(1), event: { $0 == 7 }) { g in33 die.sample(using: &g) + die.sample(using: &g)34 }35 #expect(abs(dice.mean - 7) < 4 * dice.standardError)36 #expect(close(dice.standardDeviation, (35.0 / 6).squareRoot(), rtol: 0.01))37 #expect(abs(try #require(dice.eventProbability) - 1.0 / 6) < 4 * (try #require(dice.eventStandardError)))38 #expect(dice.quantiles.map(\.value) == [5, 7, 9] && dice.minimum == 2 && dice.maximum == 12)39 #expect(dice.histogram.counts.count == 11 && dice.histogram.total == 120_000)40 #expect(dice.meanLower < 7 && dice.meanUpper > 7)41 #expect(close(dice.probability(where: { $0 >= 10 }), 6.0 / 36, rtol: 0.03))42 #expect(try dice.quantile(0.5) == 7)43 #expect(dice.verification.isVerified)44 }4546 @Test func sameSeedSameResultDifferentSeedDifferentResult() throws {47 let normal = try NormalDistribution(mean: 100, standardDeviation: 15)48 func run(_ seed: UInt64) throws -> MonteCarloSummary { try MonteCarlo.run(trials: 5000, seed: seed) { g in normal.sample(using: &g) } }49 let a = try run(2026), b = try run(2026), c = try run(2027)50 #expect(a == b) // every statistic and every outcome, bit for bit51 #expect(a.values != c.values && a.mean != c.mean)52 #expect(try MonteCarlo.summarize(a.values, seed: 2026) == a)53 }5455 @Test func asyncRunMatchesSyncAndReportsProgress() async throws {56 let log = ProgressLog()57 let exponential = try ExponentialDistribution(rate: 0.5)58 let sync = try MonteCarlo.run(trials: 10_000, seed: 99, event: { $0 > 4 }) { g in exponential.sample(using: &g) }59 let asyncResult = try await MonteCarlo.runAsync(trials: 10_000, seed: 99, progressInterval: 2500, event: { $0 > 4 }, progress: { log.add($0) }) { g in60 exponential.sample(using: &g)61 }62 #expect(asyncResult == sync)63 #expect(log.all.map(\.completed) == [2500, 5000, 7500, 10_000] && log.all.last?.fraction == 1)64 #expect(abs(try #require(sync.eventProbability) - exp(-2)) < 4 * (try #require(sync.eventStandardError)))65 var calls = 066 _ = try MonteCarlo.run(trials: 1050, seed: 1, progressInterval: 500, progress: { _ in calls += 1 }) { g in Sampling.uniform(using: &g) }67 #expect(calls == 3) // 500, 1000 and the final 105068 }6970 @Test func asyncRunIsCancellable() async throws {71 let task = Task {72 try await MonteCarlo.runAsync(trials: 40_000_000, seed: 5, progressInterval: 1000) { g in Sampling.standardNormal(using: &g) }73 }74 try await Task.sleep(nanoseconds: 30_000_000)75 task.cancel()76 let outcome = await task.result77 switch outcome {78 case .success: Issue.record("a cancelled simulation must not return a summary")79 case .failure(let error): #expect(error is CancellationError)80 }81 // Already-cancelled tasks stop before the first trial.82 let early = Task { () -> MonteCarloSummary in83 withUnsafeCurrentTask { $0?.cancel() }84 return try await MonteCarlo.runAsync(trials: 10, seed: 1) { g in Sampling.uniform(using: &g) }85 }86 #expect(await { if case .failure(let e) = await early.result { return e is CancellationError }; return false }())87 }8889 @Test func errorsAndWarnings() throws {90 #expect(throws: StatisticsError.self) { try MonteCarlo.run(trials: 0, seed: 1) { _ in 1 } }91 #expect(throws: StatisticsError.probabilityOutOfRange(2)) { try MonteCarlo.run(trials: 10, seed: 1, quantiles: [2]) { _ in 1 } }92 #expect(throws: StatisticsError.nonFiniteValue("The outcome of trial 3")) {93 var k = 094 return try MonteCarlo.run(trials: 10, seed: 1) { _ in k += 1; return k == 3 ? .nan : 1 }95 }96 #expect(throws: StatisticsError.emptyData) { try MonteCarlo.summarize([]) }97 // A trial closure may throw its own errors; they propagate.98 struct Stop: Error {}99 #expect(throws: Stop.self) { try MonteCarlo.run(trials: 10, seed: 1) { _ in throw Stop() } }100 let few = try MonteCarlo.run(trials: 50, seed: 3, event: { $0 > 0.99 }) { g in Sampling.uniform(using: &g) }101 #expect(few.verification.warnings.count == 2) // fewer than 1000 trials, and a rare event seen too rarely102 let one = try MonteCarlo.run(trials: 1, seed: 3) { _ in 4.5 }103 #expect(one.standardDeviation == 0 && one.mean == 4.5 && one.histogram.total == 1)104 }105106 @Test func samplingDistributionOfTheMean() throws {107 // Means of n = 25 from an exponential(1): centre 1, standard error 1/√25 = 0.2 (central limit theorem).108 let d = try ExponentialDistribution(rate: 1)109 let s = try MonteCarlo.samplingDistribution(of: { try Descriptive.mean($0) }, sampleSize: 25, replications: 20_000, seed: 11, from: d)110 #expect(abs(s.mean - 1) < 4 * s.standardError && close(s.standardDeviation, 0.2, rtol: 0.03))111 #expect(s.quantiles.map(\.probability) == [0.025, 0.25, 0.5, 0.75, 0.975])112 let population = [2.0, 4, 4, 4, 5, 5, 7, 9] // σ = 2, μ = 5113 let f = try MonteCarlo.samplingDistribution(of: { try Descriptive.mean($0) }, sampleSize: 16, replications: 20_000, seed: 12, fromPopulation: population)114 #expect(abs(f.mean - 5) < 4 * f.standardError && close(f.standardDeviation, 0.5, rtol: 0.03))115 #expect(throws: StatisticsError.self) { try MonteCarlo.samplingDistribution(of: { $0[0] }, sampleSize: 0, replications: 10, seed: 1, from: d) }116 #expect(throws: StatisticsError.emptyData) { try MonteCarlo.samplingDistribution(of: { $0[0] }, sampleSize: 2, replications: 10, seed: 1, fromPopulation: []) }117 }118}119120@Suite struct ScenarioTests {121 static func loan(_ inputs: [String: Double]) throws -> [String: Double] {122 let principal = inputs["principal"]!, rate = inputs["rate"]! / 12, n = inputs["years"]! * 12123 let payment = principal * rate / (1 - pow(1 + rate, -n))124 return ["payment": payment, "totalInterest": payment * n - principal]125 }126127 @Test func scenarioTable() throws {128 let table = try ScenarioAnalysis.evaluate([129 Scenario(name: "Base", inputs: ["principal": 300_000, "rate": 0.05, "years": 25]),130 Scenario(name: "Rates up", inputs: ["principal": 300_000, "rate": 0.07, "years": 25]),131 Scenario(name: "Shorter", inputs: ["principal": 300_000, "rate": 0.05, "years": 15]),132 ], model: Self.loan)133 #expect(table.inputNames == ["principal", "rate", "years"] && table.outputNames == ["payment", "totalInterest"])134 #expect(close(try #require(table.output("payment", in: "Base")), 1753.7701246, rtol: 1e-9))135 #expect(try #require(table.output("payment", in: "Rates up")) > 1753.78 && table.output("nothing", in: "Base") == nil)136 #expect(table.column("payment")?.count == 3 && table.changesFromBase[0].outputs == [0, 0])137 #expect(close(table.changesFromBase[1].inputs[1], 0.02, rtol: 1e-12) && table.changesFromBase[2].outputs[1] < 0)138 let ordered = try ScenarioAnalysis.evaluate([Scenario(name: "Base", inputs: ["principal": 1000, "rate": 0.05, "years": 1])], outputs: ["totalInterest"], model: Self.loan)139 #expect(ordered.outputNames == ["totalInterest"])140 }141142 @Test func oneWaySensitivity() throws {143 let base = ["principal": 300_000.0, "rate": 0.05, "years": 25]144 let t = try ScenarioAnalysis.sensitivity(base: base, varying: "rate", values: [0.03, 0.05, 0.07], model: Self.loan)145 let payments = try #require(t.column("payment"))146 #expect(payments == payments.sorted() && t.rows.count == 3 && t.rows[1].inputs == [300_000, 0.05, 25])147 #expect(throws: StatisticsError.self) { try ScenarioAnalysis.sensitivity(base: base, varying: "tax", values: [1], model: Self.loan) }148 #expect(throws: StatisticsError.emptyData) { try ScenarioAnalysis.sensitivity(base: base, varying: "rate", values: [], model: Self.loan) }149 }150151 @Test func scenarioErrors() {152 #expect(throws: StatisticsError.emptyData) { try ScenarioAnalysis.evaluate([], model: Self.loan) }153 #expect(throws: StatisticsError.self) {154 try ScenarioAnalysis.evaluate([Scenario(name: "A", inputs: ["x": 1]), Scenario(name: "A", inputs: ["x": 2])]) { ["y": $0["x"]!] }155 }156 #expect(throws: StatisticsError.self) {157 try ScenarioAnalysis.evaluate([Scenario(name: "A", inputs: ["x": 1]), Scenario(name: "B", inputs: ["z": 2])]) { _ in ["y": 1] }158 }159 #expect(throws: StatisticsError.self) { try ScenarioAnalysis.evaluate([Scenario(name: "A", inputs: ["x": 0])]) { ["y": 1 / $0["x"]!] } }160 #expect(throws: StatisticsError.self) { try ScenarioAnalysis.evaluate([Scenario(name: "A", inputs: ["x": 0])], outputs: ["missing"]) { _ in ["y": 1] } }161 #expect(throws: StatisticsError.self) { try ScenarioAnalysis.evaluate([Scenario(name: "A", inputs: ["x": .nan])]) { _ in ["y": 1] } }162 }163}164165@Suite struct BootstrapTests {166 @Test func intervalArithmeticMatchesTheReferenceOnFixedReplicates() throws {167 // Same replicates as the Python reference, so the three interval formulas are compared exactly (no random stream involved).168 let cases = try Reference.cases("bootstrap")169 #expect(cases.count == 6)170 for c in cases {171 let reps = c.doubles("replicates"), theta = c.double("estimate"), level = c.double("level")172 let pct = try Bootstrap.interval(replicates: reps, estimate: theta, level: level, method: .percentile)173 #expect(close(pct.lower, c.doubles("percentile")[0], rtol: 1e-12) && close(pct.upper, c.doubles("percentile")[1], rtol: 1e-12))174 let basic = try Bootstrap.interval(replicates: reps, estimate: theta, level: level, method: .basic)175 #expect(close(basic.lower, c.doubles("basic")[0], rtol: 1e-12) && close(basic.upper, c.doubles("basic")[1], rtol: 1e-12))176 let bca = try Bootstrap.interval(replicates: reps, estimate: theta, jackknife: c.doubles("jackknife"), level: level, method: .bca)177 #expect(close(try #require(bca.biasCorrection), c.double("z0"), rtol: 1e-10, atol: 1e-12), "\(c.string("name")) z0")178 #expect(close(try #require(bca.acceleration), c.double("acceleration"), rtol: 1e-10, atol: 1e-14), "\(c.string("name")) a")179 #expect(close(bca.lower, c.doubles("bca")[0], rtol: 1e-9) && close(bca.upper, c.doubles("bca")[1], rtol: 1e-9), "\(c.string("name")) BCa")180 }181 }182183 @Test func endToEndAgreesWithScipyBootstrap() throws {184 // Different random streams, so agreement is statistical: limits within 6% of the interval width of scipy's 20 000-resample result.185 for c in try Reference.cases("bootstrap") where c.has("scipy_bca") {186 let data = c.doubles("data"), name = c.string("name")187 let statistic: ([Double]) throws -> Double = switch name {188 case "lognormalMean": { try Descriptive.mean($0) }189 case "normalSD": { try Descriptive.standardDeviation($0) }190 default: { try Descriptive.median($0) }191 }192 for (method, key) in [(BootstrapMethod.percentile, "scipy_percentile"), (.basic, "scipy_basic"), (.bca, "scipy_bca")] {193 let r = try Bootstrap.confidenceInterval(data: data, level: 0.95, resamples: 20_000, seed: 2026, method: method, statistic: statistic)194 let e = c.doubles(key), width = e[1] - e[0]195 #expect(abs(r.lower - e[0]) < 0.06 * width && abs(r.upper - e[1]) < 0.06 * width, "\(name) \(method): (\(r.lower), \(r.upper)) vs \(e)")196 #expect(close(r.standardError, c.double("scipy_se"), rtol: 0.04), "\(name) se \(r.standardError) vs \(c.double("scipy_se"))")197 #expect(r.estimate == c.double("estimate") || close(r.estimate, c.double("estimate"), rtol: 1e-13))198 #expect(r.lower < r.upper && r.replicates.count == 20_000)199 }200 }201 }202203 @Test func seedReproducibilityAndDefaults() throws {204 let data = [12.1, 9.8, 15.3, 11.0, 10.4, 13.7, 9.1, 14.9, 12.5, 11.8, 16.2, 10.9, 13.1, 12.0, 11.4, 9.9, 14.2]205 let a = try Bootstrap.confidenceInterval(data: data, seed: 5) { try Descriptive.mean($0) }206 let b = try Bootstrap.confidenceInterval(data: data, seed: 5) { try Descriptive.mean($0) }207 let c = try Bootstrap.confidenceInterval(data: data, seed: 6) { try Descriptive.mean($0) }208 #expect(a == b && a.replicates != c.replicates)209 #expect(a.method == .bca && a.replicates.count == 2000 && a.level == 0.95 && a.seed == 5) // documented defaults210 #expect(a.biasCorrection != nil && a.acceleration != nil && a.verification.isVerified)211 #expect(a.lower < a.estimate && a.estimate < a.upper)212 // For the mean, the bootstrap standard error estimates s·√((n−1)/n)/√n and the interval resembles the t interval.213 let n = Double(data.count)214 #expect(close(a.standardError, try Descriptive.standardDeviation(data, sample: false) / n.squareRoot(), rtol: 0.05))215 let t = try ConfidenceIntervals.meanT(data: data)216 #expect(abs(a.lower - t.lower) < 0.25 * t.marginOfError && abs(a.upper - t.upper) < 0.25 * t.marginOfError)217 #expect(abs(a.bias) < 3 * a.standardError / (2000.0).squareRoot() + 1e-12)218 #expect(a.explanation.plainText.contains("Bootstrap confidence interval (BCa)") && a.explanation.inputs.contains { $0.label == "seed" && $0.value == "5" })219 }220221 @Test func pairedDataThroughTheIndexForm() throws {222 // Correlation needs rows resampled together: the index-based entry point does that.223 let x = [1.0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]224 let y = [2.1, 2.9, 4.2, 3.8, 6.1, 5.2, 8.3, 7.1, 9.9, 9.2, 12.4, 10.8, 13.9, 13.1, 16.5, 15.2]225 let r = try Bootstrap.confidenceInterval(count: x.count, resamples: 3000, seed: 77, method: .percentile) { idx in226 try Descriptive.correlation(idx.map { x[$0] }, idx.map { y[$0] })227 }228 #expect(close(r.estimate, try Descriptive.correlation(x, y), rtol: 1e-14) && r.lower > 0.9 && r.upper <= 1 && r.lower < r.estimate)229 }230231 @Test func errorPaths() {232 let data = [1.0, 2, 3, 4, 5, 6, 7, 8, 9, 10]233 #expect(throws: StatisticsError.emptyData) { try Bootstrap.confidenceInterval(data: [], seed: 1) { $0[0] } }234 #expect(throws: StatisticsError.insufficientSample(required: 2, actual: 1)) { try Bootstrap.confidenceInterval(data: [4], seed: 1) { $0[0] } }235 #expect(throws: StatisticsError.self) { try Bootstrap.confidenceInterval(data: data, resamples: 10, seed: 1) { $0[0] } }236 #expect(throws: StatisticsError.probabilityOutOfRange(1)) { try Bootstrap.confidenceInterval(data: data, level: 1, seed: 1) { $0[0] } }237 // A statistic that is not finite on some resample (variance-free resample → correlation undefined) is reported, not hidden.238 #expect(throws: StatisticsError.self) { try Bootstrap.confidenceInterval(data: [1, 1, 1, 2], resamples: 500, seed: 1) { try Descriptive.zScores($0)[0] } }239 // BCa without jackknife values, and BCa when every replicate is on one side of the estimate.240 #expect(throws: StatisticsError.self) { try Bootstrap.interval(replicates: data, estimate: 5, method: .bca) }241 #expect(throws: StatisticsError.self) { try Bootstrap.interval(replicates: data, estimate: 50, jackknife: data, method: .bca) }242 #expect(throws: StatisticsError.self) { try Bootstrap.interval(replicates: [1], estimate: 1, method: .percentile) }243 // A constant statistic has no BCa interval (all replicates tie with the estimate → share = ½, a = 0 → degenerate but defined).244 let constant = try? Bootstrap.confidenceInterval(data: data, resamples: 200, seed: 1, method: .percentile) { _ in 3 }245 #expect(constant?.lower == 3 && constant?.upper == 3 && constant?.verification.status == .warning)246 }247}248