SPB Git

Repository Public MIT

SimulationTests.swift 14.8 KB 258 lines swift

1// ORIX Calc2// File: Packages/ORIXFeatureStatistics/Tests/ORIXFeatureStatisticsTests/SimulationTests.swift3// Description: Tests of the Simulation section — seeded reproducibility, agreement with the engine, real cancellation, scenarios, sampling distributions, bootstrap.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Testing9import ORIXExpression10import ORIXStatistics11import ORIXUI12@testable import ORIXFeatureStatistics1314@MainActor @Suite struct SimulationTests {15    private var context: EvalContext { EvalContext() }1617    @Test func monteCarloIsReproducibleWithItsSeed() async throws {18        let m = SimulationModel()19        m.state.trials = 20_000; m.state.seed = 4220        await m.runMonteCarlo(context: context)21        let first = try #require(m.monteCarlo)22        await m.runMonteCarlo(context: context)23        let second = try #require(m.monteCarlo)24        #expect(first == second, "same seed, identical results (every value)")25        #expect(first.summary.seed == 42 && first.summary.trials == 20_000 && first.outputName == "profit")26        m.state.seed = 4327        await m.runMonteCarlo(context: context)28        let third = try #require(m.monteCarlo)29        #expect(third.summary.mean != first.summary.mean)30        #expect(m.running == nil && m.errors.isEmpty)3132        // Independent check: E[profit] = E[units]·(E[price] − E[cost]) − E[fixed] = 1000·(10 − 6) − 3000 for independent inputs.33        let s = first.summary34        #expect(abs(s.mean - 1000) < 5 * s.standardError, "mean \(s.mean), SE \(s.standardError)")35        #expect(s.standardError == s.standardDeviation / Double(s.trials).squareRoot())36        // P(profit < 0) reported with the threshold of the state.37        let p = try #require(s.eventProbability)38        #expect(p == Double(s.values.filter { $0 < 0 }.count) / Double(s.trials))39        #expect(s.quantiles.map(\.probability) == SimulationEngine.quantiles)40        #expect(s.histogram.total == s.trials)41    }4243    @Test func compiledModelAgreesWithTheEngineRunDirectly() async throws {44        let state = SimState()45        let prepared = try SimulationEngine.prepare(expression: state.expression, inputs: state.inputs, context: context)46        let viaModel = try await SimulationEngine.monteCarlo(prepared, trials: 5000, seed: 7, threshold: 0)47        let units = try NormalDistribution(mean: 1000, standardDeviation: 150), price = try UniformDistribution(lowerBound: 9, upperBound: 11)48        let cost = try NormalDistribution(mean: 6, standardDeviation: 0.4), fixed = try UniformDistribution(lowerBound: 2500, upperBound: 3500)49        let direct = try MonteCarlo.run(trials: 5000, seed: 7, quantiles: SimulationEngine.quantiles, event: { $0 < 0 }) { g in50            let u = units.sample(using: &g), p = price.sample(using: &g), c = cost.sample(using: &g), f = fixed.sample(using: &g)51            return u * (p - c) - f52        }53        #expect(viaModel.values.count == 5000)54        #expect(zip(viaModel.values, direct.values).allSatisfy { close($0, $1, 1e-12) }, "the expression engine computes the same outcomes, draw for draw")55        #expect(close(viaModel.mean, direct.mean, 1e-12) && viaModel.eventProbability == direct.eventProbability)56    }5758    @Test func cancellationStopsARunningSimulation() async throws {59        let m = SimulationModel()60        m.state.trials = 5_000_000; m.state.seed = 161        let run = Task { await m.runMonteCarlo(context: EvalContext()) }62        // Wait until the job reports progress, then cancel it.63        var waited = 064        while m.progress == 0, waited < 2000 {65            if m.running == nil, waited > 5 { break }66            try await Task.sleep(for: .milliseconds(5)); waited += 167        }68        #expect(m.running == .monteCarlo, "the simulation is still running after \(waited * 5) ms")69        m.cancel()70        #expect(m.running == nil)71        await run.value72        #expect(m.monteCarlo == nil, "a cancelled run publishes nothing")73        #expect(m.cancelledNotice == .monteCarlo)74        #expect(m.errors[.monteCarlo] == nil)7576        // At the engine level the cancellation is real: the detached job throws CancellationError.77        let prepared = try SimulationEngine.prepare(expression: m.state.expression, inputs: m.state.inputs, context: EvalContext())78        let job = Task.detached { try await SimulationEngine.monteCarlo(prepared, trials: 20_000_000, seed: 1, threshold: nil) }79        try await Task.sleep(for: .milliseconds(30))80        job.cancel()81        let started = ContinuousClock.now82        await #expect(throws: CancellationError.self) { try await job.value }83        #expect(ContinuousClock.now - started < .seconds(5), "it stops promptly instead of finishing 20 million trials")8485        // A new run after a cancel works normally.86        m.state.trials = 200087        await m.runMonteCarlo(context: EvalContext())88        #expect(m.monteCarlo?.summary.trials == 2000 && m.cancelledNotice == nil)89    }9091    @Test func modelErrorsAreExplained() async throws {92        let m = SimulationModel()93        m.state.expression = "units * margin"94        await m.runMonteCarlo(context: context)95        #expect(m.monteCarlo == nil && m.errors[.monteCarlo]?.contains("margin") == true, "unknown variable named")96        m.state.expression = "units *"97        await m.runMonteCarlo(context: context)98        #expect(m.errors[.monteCarlo]?.isEmpty == false)99        m.state.expression = ""100        await m.runMonteCarlo(context: context)101        #expect(m.errors[.monteCarlo]?.contains("Enter the output expression") == true)102        m.state.expression = "units + price"103        m.state.inputs[1].name = "units"104        await m.runMonteCarlo(context: context)105        #expect(m.errors[.monteCarlo]?.contains("same name") == true)106        m.state.inputs[1].name = "2nd"107        await m.runMonteCarlo(context: context)108        #expect(m.errors[.monteCarlo]?.contains("not a valid input name") == true)109        m.state.inputs[1].name = "price"110        m.state.inputs[0].spec["sd"] = -5111        await m.runMonteCarlo(context: context)112        #expect(m.errors[.monteCarlo]?.hasPrefix("Input units:") == true, "the engine's parameter error, with the input's name")113        m.state.inputs[0].spec["sd"] = 150114        m.state.trials = 0115        await m.runMonteCarlo(context: context)116        #expect(m.errors[.monteCarlo]?.contains("at least 1") == true)117        // A model that is undefined for some draws is the engine's error, not a silent NaN.118        m.state.trials = 2000; m.state.expression = "sqrt(units - 1000)"119        await m.runMonteCarlo(context: context)120        #expect(m.monteCarlo == nil && m.errors[.monteCarlo]?.contains("finite") == true)121        // Bare expression, calculator variables and functions.122        var ctx = EvalContext()123        ctx.variables["k"] = Value(2.0)124        m.state.expression = "k * max(units, 900)"125        await m.runMonteCarlo(context: ctx)126        #expect(m.monteCarlo?.outputName == "output" && (m.monteCarlo?.summary.minimum ?? 0) >= 1800)127    }128129    @Test func scenarios() throws {130        let m = SimulationModel()131        m.runScenarios(context: context)132        let o = try #require(m.scenarioOutput)133        #expect(o.outputName == "profit" && o.table.rows.count == 3)134        #expect(o.table.inputNames == ["cost", "fixed", "price", "units"])135        // Base: the means. 1000·(10 − 6) − 3000 = 1000.136        #expect(close(o.table.rows[0].outputs[0], 1000, 1e-12))137        #expect(o.changes[0] == 0)138        // Low: every input at its 10th percentile (computed here from the engine's quantiles).139        let q = try [NormalDistribution(mean: 1000, standardDeviation: 150).quantile(0.1), UniformDistribution(lowerBound: 9, upperBound: 11).quantile(0.1),140                     NormalDistribution(mean: 6, standardDeviation: 0.4).quantile(0.1), UniformDistribution(lowerBound: 2500, upperBound: 3500).quantile(0.1)]141        #expect(close(o.table.rows[1].outputs[0], q[0] * (q[1] - q[2]) - q[3], 1e-12))142        #expect(close(o.changes[1], o.table.rows[1].outputs[0] - 1000, 1e-12))143        // Editing a scenario and missing values.144        m.state.scenarios[0].values[m.state.inputs[0].id.uuidString] = 1200145        m.runScenarios(context: context)146        #expect(close(try #require(m.scenarioOutput).table.rows[0].outputs[0], 1200 * 4 - 3000, 1e-12))147        m.state.scenarios[2].values[m.state.inputs[1].id.uuidString] = nil148        m.runScenarios(context: context)149        #expect(m.scenarioOutput == nil && m.errors[.scenarios]?.contains("enter a value for price") == true)150        m.resetScenarios()151        m.addScenario()152        m.state.scenarios[3].name = "Base"153        m.runScenarios(context: context)154        #expect(m.errors[.scenarios] == StatisticsError.invalidArgument("Scenario names must be unique.").description)155        // Removing an input removes it from the scenarios.156        m.removeScenario(m.state.scenarios[3].id)157        m.state.expression = "units * price"158        m.removeInput(m.state.inputs[3].id); m.removeInput(m.state.inputs[2].id)159        m.runScenarios(context: context)160        #expect(m.scenarioOutput?.table.inputNames == ["price", "units"])161    }162163    @Test func samplingDistributionMatchesTheEngineAndTheTheory() async throws {164        let m = SimulationModel()165        m.state.population = DistSpec(.exponential, ["rate": 0.5]); m.state.samplingStatistic = .mean166        m.state.sampleSize = 30; m.state.replications = 4000; m.state.samplingSeed = 11167        await m.runSampling()168        let o = try #require(m.sampling)169        let engine = try MonteCarlo.samplingDistribution(of: { try Descriptive.mean($0) }, sampleSize: 30, replications: 4000, seed: 11, from: ExponentialDistribution(rate: 0.5))170        #expect(o.summary == engine, "identical to the engine's sampling-distribution routine for the same seed")171        #expect(o.theoreticalMean == 2)172        #expect(close(try #require(o.theoreticalStandardError), 2 / 30.0.squareRoot(), 1e-12))173        #expect(abs(o.summary.mean - 2) < 5 * o.summary.standardError)174        #expect(abs(o.summary.standardDeviation - 2 / 30.0.squareRoot()) < 0.03)175        #expect(o.overlay.count == 101)176        // The overlay is in count units: like the bars, its area is about (number of samples) × (bin width).177        let h = o.summary.histogram178        let area = zip(o.overlay, o.overlay.dropFirst()).map { 0.5 * ($0.y + $1.y) * ($1.x - $0.x) }.reduce(0, +)179        #expect(h.binWidth > 0)180        #expect(abs(area - 4000 * h.binWidth) < 0.05 * 4000 * h.binWidth)181        await m.runSampling()182        #expect(m.sampling == o, "reproducible")183        // Median: no overlay, and the reason is stated.184        m.state.samplingStatistic = .median185        await m.runSampling()186        #expect(m.sampling?.overlay.isEmpty == true && m.sampling?.overlayNote.contains("No normal overlay") == true)187        // Cauchy population: the central limit theorem does not apply.188        m.state.samplingStatistic = .mean; m.state.population = DistSpec(.studentT, ["df": 1])189        await m.runSampling()190        #expect(m.sampling?.theoreticalMean == nil && m.sampling?.overlayNote.contains("no finite variance") == true)191        m.state.sampleSize = 0192        await m.runSampling()193        #expect(m.sampling == nil && m.errors[.sampling]?.contains("at least 1") == true)194    }195196    @Test func bootstrapMatchesTheEngineAndCanBeCancelled() async throws {197        let ds = SampleDatasets.examScores198        let m = SimulationModel()199        m.state.bootColumn = 0; m.state.bootStatistic = .median; m.state.resamples = 1500; m.state.bootSeed = 5200        #expect(m.state.bootMethod == .bca, "BCa is the default, as in the engine")201        await m.runBootstrap(dataset: ds)202        let o = try #require(m.bootstrap)203        let engine = try Bootstrap.confidenceInterval(data: ds.values(0), level: 0.95, resamples: 1500, seed: 5, method: .bca) { try Descriptive.median($0) }204        #expect(o.result.lower == engine.lower && o.result.upper == engine.upper && o.result.replicates == engine.replicates)205        #expect(o.result.method == .bca && o.result.seed == 5 && o.column == "Score")206        #expect(o.result.lower <= o.result.estimate && o.result.estimate <= o.result.upper)207        #expect(o.histogram?.total == 1500)208        for method in [BootstrapMethod.percentile, .basic] {209            m.state.bootMethod = method210            await m.runBootstrap(dataset: ds)211            let r = try #require(m.bootstrap).result212            let e = try Bootstrap.confidenceInterval(data: ds.values(0), level: 0.95, resamples: 1500, seed: 5, method: method) { try Descriptive.median($0) }213            #expect(r.method == method && r.lower == e.lower && r.upper == e.upper)214        }215        #expect(BootstrapMethod.bca.title.contains("BCa") && BootstrapMethod.basic.title.contains("Basic") && BootstrapMethod.percentile.title == "Percentile")216        m.state.resamples = 10217        await m.runBootstrap(dataset: ds)218        #expect(m.bootstrap == nil && m.errors[.bootstrap]?.contains("100") == true, "the engine's minimum is reported")219        m.state.resamples = 1500; m.state.bootColumn = 3220        await m.runBootstrap(dataset: ds)221        #expect(m.errors[.bootstrap]?.isEmpty == false)222223        // Cancellation: a long bootstrap stops at the next resample.224        let big = Dataset(title: "big", names: ["X"], columns: [(1...4000).map { Double($0 % 97) }])225        m.state.bootColumn = 0; m.state.resamples = 200_000; m.state.bootMethod = .percentile226        let run = Task { await m.runBootstrap(dataset: big) }227        var waited = 0228        while m.running == nil, waited < 400 { try await Task.sleep(for: .milliseconds(5)); waited += 1 }229        try await Task.sleep(for: .milliseconds(40))230        #expect(m.running == .bootstrap)231        let started = ContinuousClock.now232        m.cancel()233        await run.value234        #expect(ContinuousClock.now - started < .seconds(5))235        #expect(m.bootstrap == nil && m.cancelledNotice == .bootstrap && m.running == nil)236    }237238    @Test func runsAreLoggedWithTheirSeed() async throws {239        let app = makeApp()240        let store = StatisticsStore()241        store.data.load(SampleDatasets.examScores)242        store.simulation.state.trials = 3000; store.simulation.state.seed = 99243        await store.runMonteCarlo(app)244        #expect(app.history.first?.input.contains("seed 99") == true && app.history.first?.mode == .statistics)245        store.runScenarios(app)246        #expect(app.history.first?.input.hasPrefix("Scenarios:") == true)247        store.simulation.state.replications = 500248        await store.runSampling(app)249        #expect(app.history.first?.input.hasPrefix("Sampling distribution") == true)250        store.simulation.state.resamples = 300251        await store.runBootstrap(app)252        #expect(app.history.first?.input.hasPrefix("Bootstrap BCa interval") == true)253        #expect(app.history.count == 4)254        store.simulation.newSeed(for: .monteCarlo)255        #expect(store.simulation.state.seed != 99)256    }257}258
Files main
Menu

Theme