Repository Public MIT
1// ORIX Calc2// File: Packages/ORIXFeatureStatistics/Tests/ORIXFeatureStatisticsTests/SimulationTests.swift3// Description: Tests of the Simulation section — seeded reproducibility, agreement with the engine, real cancellation, scenarios, sampling distributions, bootstrap.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Testing9import ORIXExpression10import ORIXStatistics11import ORIXUI12@testable import ORIXFeatureStatistics1314@MainActor @Suite struct SimulationTests {15 private var context: EvalContext { EvalContext() }1617 @Test func monteCarloIsReproducibleWithItsSeed() async throws {18 let m = SimulationModel()19 m.state.trials = 20_000; m.state.seed = 4220 await m.runMonteCarlo(context: context)21 let first = try #require(m.monteCarlo)22 await m.runMonteCarlo(context: context)23 let second = try #require(m.monteCarlo)24 #expect(first == second, "same seed, identical results (every value)")25 #expect(first.summary.seed == 42 && first.summary.trials == 20_000 && first.outputName == "profit")26 m.state.seed = 4327 await m.runMonteCarlo(context: context)28 let third = try #require(m.monteCarlo)29 #expect(third.summary.mean != first.summary.mean)30 #expect(m.running == nil && m.errors.isEmpty)3132 // Independent check: E[profit] = E[units]·(E[price] − E[cost]) − E[fixed] = 1000·(10 − 6) − 3000 for independent inputs.33 let s = first.summary34 #expect(abs(s.mean - 1000) < 5 * s.standardError, "mean \(s.mean), SE \(s.standardError)")35 #expect(s.standardError == s.standardDeviation / Double(s.trials).squareRoot())36 // P(profit < 0) reported with the threshold of the state.37 let p = try #require(s.eventProbability)38 #expect(p == Double(s.values.filter { $0 < 0 }.count) / Double(s.trials))39 #expect(s.quantiles.map(\.probability) == SimulationEngine.quantiles)40 #expect(s.histogram.total == s.trials)41 }4243 @Test func compiledModelAgreesWithTheEngineRunDirectly() async throws {44 let state = SimState()45 let prepared = try SimulationEngine.prepare(expression: state.expression, inputs: state.inputs, context: context)46 let viaModel = try await SimulationEngine.monteCarlo(prepared, trials: 5000, seed: 7, threshold: 0)47 let units = try NormalDistribution(mean: 1000, standardDeviation: 150), price = try UniformDistribution(lowerBound: 9, upperBound: 11)48 let cost = try NormalDistribution(mean: 6, standardDeviation: 0.4), fixed = try UniformDistribution(lowerBound: 2500, upperBound: 3500)49 let direct = try MonteCarlo.run(trials: 5000, seed: 7, quantiles: SimulationEngine.quantiles, event: { $0 < 0 }) { g in50 let u = units.sample(using: &g), p = price.sample(using: &g), c = cost.sample(using: &g), f = fixed.sample(using: &g)51 return u * (p - c) - f52 }53 #expect(viaModel.values.count == 5000)54 #expect(zip(viaModel.values, direct.values).allSatisfy { close($0, $1, 1e-12) }, "the expression engine computes the same outcomes, draw for draw")55 #expect(close(viaModel.mean, direct.mean, 1e-12) && viaModel.eventProbability == direct.eventProbability)56 }5758 @Test func cancellationStopsARunningSimulation() async throws {59 let m = SimulationModel()60 m.state.trials = 5_000_000; m.state.seed = 161 let run = Task { await m.runMonteCarlo(context: EvalContext()) }62 // Wait until the job reports progress, then cancel it.63 var waited = 064 while m.progress == 0, waited < 2000 {65 if m.running == nil, waited > 5 { break }66 try await Task.sleep(for: .milliseconds(5)); waited += 167 }68 #expect(m.running == .monteCarlo, "the simulation is still running after \(waited * 5) ms")69 m.cancel()70 #expect(m.running == nil)71 await run.value72 #expect(m.monteCarlo == nil, "a cancelled run publishes nothing")73 #expect(m.cancelledNotice == .monteCarlo)74 #expect(m.errors[.monteCarlo] == nil)7576 // At the engine level the cancellation is real: the detached job throws CancellationError.77 let prepared = try SimulationEngine.prepare(expression: m.state.expression, inputs: m.state.inputs, context: EvalContext())78 let job = Task.detached { try await SimulationEngine.monteCarlo(prepared, trials: 20_000_000, seed: 1, threshold: nil) }79 try await Task.sleep(for: .milliseconds(30))80 job.cancel()81 let started = ContinuousClock.now82 await #expect(throws: CancellationError.self) { try await job.value }83 #expect(ContinuousClock.now - started < .seconds(5), "it stops promptly instead of finishing 20 million trials")8485 // A new run after a cancel works normally.86 m.state.trials = 200087 await m.runMonteCarlo(context: EvalContext())88 #expect(m.monteCarlo?.summary.trials == 2000 && m.cancelledNotice == nil)89 }9091 @Test func modelErrorsAreExplained() async throws {92 let m = SimulationModel()93 m.state.expression = "units * margin"94 await m.runMonteCarlo(context: context)95 #expect(m.monteCarlo == nil && m.errors[.monteCarlo]?.contains("margin") == true, "unknown variable named")96 m.state.expression = "units *"97 await m.runMonteCarlo(context: context)98 #expect(m.errors[.monteCarlo]?.isEmpty == false)99 m.state.expression = ""100 await m.runMonteCarlo(context: context)101 #expect(m.errors[.monteCarlo]?.contains("Enter the output expression") == true)102 m.state.expression = "units + price"103 m.state.inputs[1].name = "units"104 await m.runMonteCarlo(context: context)105 #expect(m.errors[.monteCarlo]?.contains("same name") == true)106 m.state.inputs[1].name = "2nd"107 await m.runMonteCarlo(context: context)108 #expect(m.errors[.monteCarlo]?.contains("not a valid input name") == true)109 m.state.inputs[1].name = "price"110 m.state.inputs[0].spec["sd"] = -5111 await m.runMonteCarlo(context: context)112 #expect(m.errors[.monteCarlo]?.hasPrefix("Input units:") == true, "the engine's parameter error, with the input's name")113 m.state.inputs[0].spec["sd"] = 150114 m.state.trials = 0115 await m.runMonteCarlo(context: context)116 #expect(m.errors[.monteCarlo]?.contains("at least 1") == true)117 // A model that is undefined for some draws is the engine's error, not a silent NaN.118 m.state.trials = 2000; m.state.expression = "sqrt(units - 1000)"119 await m.runMonteCarlo(context: context)120 #expect(m.monteCarlo == nil && m.errors[.monteCarlo]?.contains("finite") == true)121 // Bare expression, calculator variables and functions.122 var ctx = EvalContext()123 ctx.variables["k"] = Value(2.0)124 m.state.expression = "k * max(units, 900)"125 await m.runMonteCarlo(context: ctx)126 #expect(m.monteCarlo?.outputName == "output" && (m.monteCarlo?.summary.minimum ?? 0) >= 1800)127 }128129 @Test func scenarios() throws {130 let m = SimulationModel()131 m.runScenarios(context: context)132 let o = try #require(m.scenarioOutput)133 #expect(o.outputName == "profit" && o.table.rows.count == 3)134 #expect(o.table.inputNames == ["cost", "fixed", "price", "units"])135 // Base: the means. 1000·(10 − 6) − 3000 = 1000.136 #expect(close(o.table.rows[0].outputs[0], 1000, 1e-12))137 #expect(o.changes[0] == 0)138 // Low: every input at its 10th percentile (computed here from the engine's quantiles).139 let q = try [NormalDistribution(mean: 1000, standardDeviation: 150).quantile(0.1), UniformDistribution(lowerBound: 9, upperBound: 11).quantile(0.1),140 NormalDistribution(mean: 6, standardDeviation: 0.4).quantile(0.1), UniformDistribution(lowerBound: 2500, upperBound: 3500).quantile(0.1)]141 #expect(close(o.table.rows[1].outputs[0], q[0] * (q[1] - q[2]) - q[3], 1e-12))142 #expect(close(o.changes[1], o.table.rows[1].outputs[0] - 1000, 1e-12))143 // Editing a scenario and missing values.144 m.state.scenarios[0].values[m.state.inputs[0].id.uuidString] = 1200145 m.runScenarios(context: context)146 #expect(close(try #require(m.scenarioOutput).table.rows[0].outputs[0], 1200 * 4 - 3000, 1e-12))147 m.state.scenarios[2].values[m.state.inputs[1].id.uuidString] = nil148 m.runScenarios(context: context)149 #expect(m.scenarioOutput == nil && m.errors[.scenarios]?.contains("enter a value for price") == true)150 m.resetScenarios()151 m.addScenario()152 m.state.scenarios[3].name = "Base"153 m.runScenarios(context: context)154 #expect(m.errors[.scenarios] == StatisticsError.invalidArgument("Scenario names must be unique.").description)155 // Removing an input removes it from the scenarios.156 m.removeScenario(m.state.scenarios[3].id)157 m.state.expression = "units * price"158 m.removeInput(m.state.inputs[3].id); m.removeInput(m.state.inputs[2].id)159 m.runScenarios(context: context)160 #expect(m.scenarioOutput?.table.inputNames == ["price", "units"])161 }162163 @Test func samplingDistributionMatchesTheEngineAndTheTheory() async throws {164 let m = SimulationModel()165 m.state.population = DistSpec(.exponential, ["rate": 0.5]); m.state.samplingStatistic = .mean166 m.state.sampleSize = 30; m.state.replications = 4000; m.state.samplingSeed = 11167 await m.runSampling()168 let o = try #require(m.sampling)169 let engine = try MonteCarlo.samplingDistribution(of: { try Descriptive.mean($0) }, sampleSize: 30, replications: 4000, seed: 11, from: ExponentialDistribution(rate: 0.5))170 #expect(o.summary == engine, "identical to the engine's sampling-distribution routine for the same seed")171 #expect(o.theoreticalMean == 2)172 #expect(close(try #require(o.theoreticalStandardError), 2 / 30.0.squareRoot(), 1e-12))173 #expect(abs(o.summary.mean - 2) < 5 * o.summary.standardError)174 #expect(abs(o.summary.standardDeviation - 2 / 30.0.squareRoot()) < 0.03)175 #expect(o.overlay.count == 101)176 // The overlay is in count units: like the bars, its area is about (number of samples) × (bin width).177 let h = o.summary.histogram178 let area = zip(o.overlay, o.overlay.dropFirst()).map { 0.5 * ($0.y + $1.y) * ($1.x - $0.x) }.reduce(0, +)179 #expect(h.binWidth > 0)180 #expect(abs(area - 4000 * h.binWidth) < 0.05 * 4000 * h.binWidth)181 await m.runSampling()182 #expect(m.sampling == o, "reproducible")183 // Median: no overlay, and the reason is stated.184 m.state.samplingStatistic = .median185 await m.runSampling()186 #expect(m.sampling?.overlay.isEmpty == true && m.sampling?.overlayNote.contains("No normal overlay") == true)187 // Cauchy population: the central limit theorem does not apply.188 m.state.samplingStatistic = .mean; m.state.population = DistSpec(.studentT, ["df": 1])189 await m.runSampling()190 #expect(m.sampling?.theoreticalMean == nil && m.sampling?.overlayNote.contains("no finite variance") == true)191 m.state.sampleSize = 0192 await m.runSampling()193 #expect(m.sampling == nil && m.errors[.sampling]?.contains("at least 1") == true)194 }195196 @Test func bootstrapMatchesTheEngineAndCanBeCancelled() async throws {197 let ds = SampleDatasets.examScores198 let m = SimulationModel()199 m.state.bootColumn = 0; m.state.bootStatistic = .median; m.state.resamples = 1500; m.state.bootSeed = 5200 #expect(m.state.bootMethod == .bca, "BCa is the default, as in the engine")201 await m.runBootstrap(dataset: ds)202 let o = try #require(m.bootstrap)203 let engine = try Bootstrap.confidenceInterval(data: ds.values(0), level: 0.95, resamples: 1500, seed: 5, method: .bca) { try Descriptive.median($0) }204 #expect(o.result.lower == engine.lower && o.result.upper == engine.upper && o.result.replicates == engine.replicates)205 #expect(o.result.method == .bca && o.result.seed == 5 && o.column == "Score")206 #expect(o.result.lower <= o.result.estimate && o.result.estimate <= o.result.upper)207 #expect(o.histogram?.total == 1500)208 for method in [BootstrapMethod.percentile, .basic] {209 m.state.bootMethod = method210 await m.runBootstrap(dataset: ds)211 let r = try #require(m.bootstrap).result212 let e = try Bootstrap.confidenceInterval(data: ds.values(0), level: 0.95, resamples: 1500, seed: 5, method: method) { try Descriptive.median($0) }213 #expect(r.method == method && r.lower == e.lower && r.upper == e.upper)214 }215 #expect(BootstrapMethod.bca.title.contains("BCa") && BootstrapMethod.basic.title.contains("Basic") && BootstrapMethod.percentile.title == "Percentile")216 m.state.resamples = 10217 await m.runBootstrap(dataset: ds)218 #expect(m.bootstrap == nil && m.errors[.bootstrap]?.contains("100") == true, "the engine's minimum is reported")219 m.state.resamples = 1500; m.state.bootColumn = 3220 await m.runBootstrap(dataset: ds)221 #expect(m.errors[.bootstrap]?.isEmpty == false)222223 // Cancellation: a long bootstrap stops at the next resample.224 let big = Dataset(title: "big", names: ["X"], columns: [(1...4000).map { Double($0 % 97) }])225 m.state.bootColumn = 0; m.state.resamples = 200_000; m.state.bootMethod = .percentile226 let run = Task { await m.runBootstrap(dataset: big) }227 var waited = 0228 while m.running == nil, waited < 400 { try await Task.sleep(for: .milliseconds(5)); waited += 1 }229 try await Task.sleep(for: .milliseconds(40))230 #expect(m.running == .bootstrap)231 let started = ContinuousClock.now232 m.cancel()233 await run.value234 #expect(ContinuousClock.now - started < .seconds(5))235 #expect(m.bootstrap == nil && m.cancelledNotice == .bootstrap && m.running == nil)236 }237238 @Test func runsAreLoggedWithTheirSeed() async throws {239 let app = makeApp()240 let store = StatisticsStore()241 store.data.load(SampleDatasets.examScores)242 store.simulation.state.trials = 3000; store.simulation.state.seed = 99243 await store.runMonteCarlo(app)244 #expect(app.history.first?.input.contains("seed 99") == true && app.history.first?.mode == .statistics)245 store.runScenarios(app)246 #expect(app.history.first?.input.hasPrefix("Scenarios:") == true)247 store.simulation.state.replications = 500248 await store.runSampling(app)249 #expect(app.history.first?.input.hasPrefix("Sampling distribution") == true)250 store.simulation.state.resamples = 300251 await store.runBootstrap(app)252 #expect(app.history.first?.input.hasPrefix("Bootstrap BCa interval") == true)253 #expect(app.history.count == 4)254 store.simulation.newSeed(for: .monteCarlo)255 #expect(store.simulation.state.seed != 99)256 }257}258