Repository Public
1import Foundation2import Testing3import AnalyticsTestSupport4@testable import Statistics56private let resamplingFixture = Fixture.load("resampling", .module)78private func sampleSD(_ v: [Double]) -> Double { Descriptive.standardDeviation(v) }9private func medianOf(_ v: [Double]) -> Double {10 let s = v.sorted()11 return s.count % 2 == 1 ? s[s.count / 2] : 0.5 * (s[s.count / 2 - 1] + s[s.count / 2])12}1314@Suite struct BootstrapTests {15 let f = resamplingFixture["bootstrap"]16 var x: [Double] { f["x"].arr }17 var y: [Double] { f["y"].arr }18 var indices: [[Int]] { f["indices"].list.map(\.ints) }1920 func check(_ r: BootstrapResult, _ c: JSON, _ label: String) {21 #expect(isClose(r.estimate, c["estimate"].d, rel: 1e-13), "\(label) estimate")22 #expect(isClose(r.standardError, c["se"].d, rel: 1e-11), "\(label) se \(r.standardError) vs \(c["se"].d)")23 #expect(isClose(r.bias, c["bias"].d, rel: 1e-9, abs: 1e-13), "\(label) bias")24 #expect(isClose(r.percentile.lower, c["pct"][0].d, rel: 1e-12), "\(label) pct lower")25 #expect(isClose(r.percentile.upper, c["pct"][1].d, rel: 1e-12), "\(label) pct upper")26 #expect(isClose(r.biasCorrection, c["z0"].d, rel: 1e-11, abs: 1e-13), "\(label) z0")27 #expect(isClose(r.acceleration, c["acc"].d, rel: 1e-9, abs: 1e-14), "\(label) acceleration")28 #expect(r.bca != nil, "\(label) bca defined")29 #expect(isClose(r.bca?.lower ?? .nan, c["bca"][0].d, rel: 1e-10), "\(label) bca lower \(r.bca?.lower ?? .nan) vs \(c["bca"][0].d)")30 #expect(isClose(r.bca?.upper ?? .nan, c["bca"][1].d, rel: 1e-10), "\(label) bca upper")31 #expect(allClose(Array(r.replicates.prefix(10)), c["reps_head"].arr, rel: 1e-12), "\(label) replicates")32 }3334 @Test func matchesSciPyOnTheSameResamples() throws {35 let data = x, sets = indices36 let stats: [String: ([Double]) -> Double] = ["mean": { Descriptive.mean($0) }, "sd": sampleSD, "median": medianOf]37 for (key, c) in f["cases"].dict where c["stat"].s != "pearson" {38 let stat = stats[c["stat"].s]!39 let r = try Bootstrap.evaluate(count: data.count, indexSets: sets, confidenceLevel: c["level"].d) { idx in stat(idx.map { data[$0] }) }40 check(r, c, key)41 #expect(r.resamples == 2000 && r.sampleSize == 25)42 }43 }4445 @Test func pairedStatisticMatchesSciPy() throws {46 let xs = x, ys = y47 let c = f["cases"]["pearson_0.9"]48 let r = try Bootstrap.evaluate(count: xs.count, indexSets: indices, confidenceLevel: 0.9) { idx in49 Descriptive.pearson(idx.map { xs[$0] }, idx.map { ys[$0] })50 }51 check(r, c, "pearson")52 }5354 @Test func seededRunsAreReproducible() throws {55 let a = try Bootstrap.run(x, resamples: 500, seed: 11) { Descriptive.mean($0) }56 let b = try Bootstrap.run(x, resamples: 500, seed: 11) { Descriptive.mean($0) }57 let c = try Bootstrap.run(x, resamples: 500, seed: 12) { Descriptive.mean($0) }58 #expect(a == b)59 #expect(a.replicates != c.replicates)60 #expect(a.seed == 11 && a.replicates.count == 500)61 #expect(a.percentile.lower < a.estimate && a.estimate < a.percentile.upper)62 #expect(a.percentile.lower < a.percentile.upper)63 // bootstrap SE of the mean ≈ s/√n · √((n−1)/n)64 let theory = sampleSD(x) / Double(x.count).squareRoot() * (Double(x.count - 1) / Double(x.count)).squareRoot()65 let big = try Bootstrap.run(x, resamples: 20_000, seed: 5) { Descriptive.mean($0) }66 #expect(isClose(big.standardError, theory, rel: 0.03))67 #expect(abs(big.bias) < 4 * theory / 20_000.0.squareRoot() + 1e-12)68 let p = try Bootstrap.runPaired(x, y, resamples: 300, seed: 3) { Descriptive.pearson($0, $1) }69 #expect(p.estimate == Descriptive.pearson(x, y))70 #expect(p.table().count >= 12)71 }7273 @Test func coverageOfTheMeanIsNearNominal() throws {74 // 300 samples of n = 30 from N(5, 2²): the 90 % intervals should cover 5 about 90 % of the time75 var g = Xoshiro256StarStar(seed: 2024)76 var hitP = 0, hitB = 077 let reps = 30078 for k in 0..<reps {79 let sample = (0..<30).map { _ in 5 + 2 * g.nextNormal() }80 let r = try Bootstrap.run(sample, resamples: 400, confidenceLevel: 0.9, seed: UInt64(k)) { Descriptive.mean($0) }81 if r.percentile.lower <= 5 && 5 <= r.percentile.upper { hitP += 1 }82 if let b = r.bca, b.lower <= 5 && 5 <= b.upper { hitB += 1 }83 }84 // binomial(300, 0.9): sd ≈ 5.2 → accept 0.82…0.96 (bootstrap intervals under-cover slightly at n = 30)85 #expect(hitP >= 246 && hitP <= 288, "percentile coverage \(hitP)/300")86 #expect(hitB >= 246 && hitB <= 288, "BCa coverage \(hitB)/300")87 }8889 @Test func invalidInputsAndDegenerateStatistics() throws {90 #expect(throws: StatError.self) { try Bootstrap.run([1.0], seed: 1) { $0[0] } }91 #expect(throws: StatError.self) { try Bootstrap.run([1.0, 2, 3], resamples: 1, seed: 1) { $0[0] } }92 #expect(throws: StatError.self) { try Bootstrap.run([1.0, 2, 3], confidenceLevel: 1.2, seed: 1) { $0[0] } }93 #expect(throws: StatError.self) { try Bootstrap.runPaired([1, 2], [1], seed: 1) { a, _ in a[0] } }94 // a constant statistic has no acceleration: BCa is reported as undefined, percentile still works95 let r = try Bootstrap.run([1, 2, 3, 4], resamples: 50, seed: 1) { _ in 7 }96 #expect(r.bca == nil && r.percentile.lower == 7 && r.percentile.upper == 7 && r.standardError == 0)97 struct Boom: Error {}98 #expect(throws: Boom.self) { try Bootstrap.run([1.0, 2, 3], resamples: 10, seed: 1) { _ in throw Boom() } }99 }100}101102@Suite struct PermutationTestTests {103 let f = resamplingFixture["permutation"]104105 @Test func exactTwoSampleMatchesSciPy() throws {106 let a = f["a"].arr, b = f["b"].arr107 for alt in Alternative.allCases {108 let c = f["two"]["mean_\(alt.rawValue)"]109 let r = try PermutationTest.twoSample(a, b, alternative: alt)110 #expect(r.exact && r.permutations == c["n"].i && r.permutations == 462)111 #expect(isClose(r.statistic, c["stat"].d, rel: 1e-13))112 #expect(isClose(r.pValue, c["p"].d, rel: 1e-13), "\(alt) p \(r.pValue) vs \(c["p"].d)")113 #expect(allClose(r.nullDistribution.sorted(), c["null_sorted"].arr, rel: 1e-11, abs: 1e-12))114 let m = f["two"]["median_\(alt.rawValue)"]115 let rm = try PermutationTest.twoSample(a, b, alternative: alt) { medianOf($0) - medianOf($1) }116 #expect(isClose(rm.statistic, m["stat"].d, rel: 1e-13))117 #expect(isClose(rm.pValue, m["p"].d, rel: 1e-13), "median \(alt) p \(rm.pValue) vs \(m["p"].d)")118 }119 }120121 @Test func exactPairedMatchesSciPy() throws {122 let before = f["before"].arr, after = f["after"].arr123 for alt in Alternative.allCases {124 let c = f["paired"]["mean_\(alt.rawValue)"]125 let r = try PermutationTest.paired(before, after, alternative: alt)126 #expect(r.exact && r.permutations == 512 && c["n"].i == 512)127 #expect(isClose(r.statistic, c["stat"].d, rel: 1e-13))128 #expect(isClose(r.pValue, c["p"].d, rel: 1e-13), "paired \(alt) p \(r.pValue) vs \(c["p"].d)")129 #expect(allClose(r.nullDistribution.sorted(), c["null_sorted"].arr, rel: 1e-11, abs: 1e-12))130 }131 }132133 @Test func monteCarloApproximatesTheExactTest() throws {134 let g1 = f["big"]["g1"].arr, g2 = f["big"]["g2"].arr135 let exact = try PermutationTest.twoSample(g1, g2, resamples: 200_000)136 #expect(exact.exact && exact.permutations == 184_756 && f["big"]["n"].i == 184_756)137 #expect(isClose(exact.pValue, f["big"]["p"].d, rel: 1e-12))138 let mc = try PermutationTest.twoSample(g1, g2, resamples: 20_000, seed: 77)139 #expect(!mc.exact && mc.permutations == 20_000 && mc.seed == 77)140 // each one-sided tail has binomial error; allow 5 standard errors of the doubled tail141 let p = exact.pValue142 let se = 2 * (p / 2 * (1 - p / 2) / 20_000).squareRoot()143 #expect(abs(mc.pValue - p) < 5 * se, "MC p \(mc.pValue) vs exact \(p)")144 #expect(try PermutationTest.twoSample(g1, g2, resamples: 20_000, seed: 77) == mc)145 #expect(try PermutationTest.twoSample(g1, g2, resamples: 20_000, seed: 78).nullDistribution != mc.nullDistribution)146 // (b+1)/(m+1): the smallest possible Monte Carlo p-value is 1/(m+1), never 0147 let far = try PermutationTest.twoSample((0..<12).map { Double($0) }, (0..<12).map { Double($0) + 100 }, alternative: .less, resamples: 999, seed: 1)148 #expect(!far.exact && isClose(far.pValue, 1.0 / 1000, rel: 1e-12))149 let pairedMC = try PermutationTest.paired((0..<20).map { Double($0) + 1 }, (0..<20).map { Double($0) }, alternative: .greater, resamples: 4999, seed: 9)150 #expect(!pairedMC.exact && isClose(pairedMC.pValue, 1.0 / 5000, rel: 1e-12))151 }152153 @Test func conventionsAndErrors() throws {154 // identical samples: the observed arrangement is the centre of the null → two-sided p = 1155 let same = try PermutationTest.twoSample([1, 2, 3], [1, 2, 3])156 #expect(same.exact && same.permutations == 20 && same.pValue == 1 && same.statistic == 0)157 // smallest exact p: only one arrangement as extreme → 1 / C(6,3) one-sided, doubled two-sided158 let sep = try PermutationTest.twoSample([10, 11, 12], [1, 2, 3], alternative: .greater)159 #expect(isClose(sep.pValue, 1.0 / 20, rel: 1e-14))160 #expect(isClose(try PermutationTest.twoSample([10, 11, 12], [1, 2, 3]).pValue, 2.0 / 20, rel: 1e-14))161 #expect(isClose(try PermutationTest.twoSample([10, 11, 12], [1, 2, 3], alternative: .less).pValue, 1, rel: 1e-14))162 let pr = try PermutationTest.paired([5, 6, 7, 8], [1, 1, 1, 1], alternative: .greater)163 #expect(pr.exact && isClose(pr.pValue, 1.0 / 16, rel: 1e-14))164 #expect(throws: StatError.self) { try PermutationTest.twoSample([], [1]) }165 #expect(throws: StatError.self) { try PermutationTest.paired([1, 2], [1]) }166 #expect(throws: StatError.self) { try PermutationTest.twoSample([1], [2], resamples: 0) }167 #expect(sep.table().count >= 7)168 }169}170171@Suite struct MonteCarloEngineTests {172 let f = resamplingFixture["montecarlo"]173174 @Test func outputSummaryMatchesNumPy() {175 let y = f["y"].arr176 let o = MonteCarlo.summarize(name: "y", samples: y, inputNames: ["a", "b", "c"], inputSamples: f["inputs"].mat)177 #expect(o.count == 3000 && o.invalidCount == 0)178 #expect(isClose(o.mean, f["mean"].d, rel: 1e-13))179 #expect(isClose(o.standardDeviation, f["sd"].d, rel: 1e-12))180 #expect(isClose(o.standardError, f["se"].d, rel: 1e-12))181 #expect(o.minimum == f["min"].d && o.maximum == f["max"].d)182 #expect(isClose(o.median, f["median"].d, rel: 1e-13))183 #expect(isClose(o.skewness, f["skew"].d, rel: 1e-10))184 #expect(isClose(o.excessKurtosis, f["kurt"].d, rel: 1e-10))185 #expect(allClose(o.percentiles.map(\.value), f["percentiles"].arr, rel: 1e-12))186 #expect(o.percentiles.map(\.probability) == f["probs"].arr)187 #expect(o.histogram.counts == f["hist_counts"].ints)188 #expect(allClose(o.histogram.edges, f["hist_edges"].arr, rel: 1e-12, abs: 1e-12))189 #expect(o.histogram.counts.reduce(0, +) == 3000)190 let conv = f["convergence"].list191 #expect(o.convergence.count == conv.count)192 for (p, c) in zip(o.convergence, conv) {193 #expect(p.trials == c[0].i)194 #expect(isClose(p.mean, c[1].d, rel: 1e-11))195 #expect(isClose(p.standardError, c[2].d, rel: 1e-10))196 }197 // sensitivities: Spearman of each input with the output, sorted by |ρ|198 let rho = f["spearman"].arr, contrib = f["contrib"].arr199 let byName = Dictionary(uniqueKeysWithValues: o.sensitivities.map { ($0.input, $0) })200 for (i, name) in ["a", "b", "c"].enumerated() {201 #expect(isClose(byName[name]!.rankCorrelation, rho[i], rel: 1e-11))202 #expect(isClose(byName[name]!.contributionToVariance, contrib[i], rel: 1e-10))203 }204 let mags = o.sensitivities.map { abs($0.rankCorrelation) }205 #expect(mags == mags.sorted(by: >))206 #expect(o.sensitivities.first?.input == "a")207 #expect(allClose(ResamplingMath.spearmanMatrix(f["inputs"].mat).flatMap { $0 }, f["input_spearman"].mat.flatMap { $0 }, rel: 1e-10, abs: 1e-13))208 #expect(o.table().count > 15)209 }210211 @Test func invalidOutcomesAreSetAside() {212 let o = MonteCarlo.summarize(name: "y", samples: [1, .nan, 3, .infinity, 5], inputNames: ["x"], inputSamples: [[1, 2, 3, 4, 5]])213 #expect(o.count == 3 && o.invalidCount == 2 && o.mean == 3 && o.minimum == 1 && o.maximum == 5)214 #expect(o.sensitivities.count == 1 && isClose(o.sensitivities[0].rankCorrelation, 1, rel: 1e-14))215 let constant = MonteCarlo.summarize(name: "c", samples: [2, 2, 2])216 #expect(constant.histogram.edges.first == 1.5 && constant.histogram.edges.last == 2.5 && constant.histogram.counts.reduce(0, +) == 3)217 #expect(MonteCarlo.summarize(name: "e", samples: []).count == 0)218 }219220 let inputs = [MonteCarloInput(name: "price", distribution: .triangular(min: 8, mode: 10, max: 13)),221 MonteCarloInput(name: "volume", distribution: .normal(mean: 1000, sd: 150)),222 MonteCarloInput(name: "cost", distribution: .lognormal(mean: 6, sd: 1))]223224 @Test func seededRunsAreBitReproducible() throws {225 var opt = MonteCarloOptions(trials: 4000, seed: 123)226 let model: ([Double]) -> Double = { ($0[0] - $0[2]) * $0[1] - 2000 }227 let a = try MonteCarlo.run(inputs: inputs, outputName: "profit", options: opt, model: model)228 let b = try MonteCarlo.run(inputs: inputs, outputName: "profit", options: opt, model: model)229 #expect(a == b)230 #expect(a.trialsCompleted == 4000 && !a.cancelled && a.outputSamples[0].count == 4000 && a.inputSamples.count == 3)231 opt.seed = 124232 let c = try MonteCarlo.run(inputs: inputs, outputName: "profit", options: opt, model: model)233 #expect(c.outputSamples != a.outputSamples)234 // outputs really are the model applied to the stored inputs235 for t in [0, 17, 3999] { #expect(a.outputSamples[0][t] == model([a.inputSamples[0][t], a.inputSamples[1][t], a.inputSamples[2][t]])) }236 // analytic mean of an independent product model: (E[price] − E[cost])·E[volume] − 2000237 let expected = ((8.0 + 10 + 13) / 3 - 6) * 1000 - 2000238 #expect(abs(a.outputs[0].mean - expected) < 5 * a.outputs[0].standardError)239 #expect(a.outputs[0].sensitivities.count == 3 && a.achievedInputRankCorrelation == nil)240 // input marginals converge to their theoretical moments241 for (j, i) in inputs.enumerated() {242 let col = a.inputSamples[j]243 #expect(abs(Descriptive.mean(col) - i.distribution.mean) < 5 * (i.distribution.variance / 4000).squareRoot(), "\(i.name)")244 }245 opt.keepSamples = false246 let lean = try MonteCarlo.run(inputs: inputs, outputName: "profit", options: opt, model: model)247 #expect(lean.inputSamples.isEmpty && lean.outputSamples.isEmpty && lean.outputs[0].count == 4000)248 #expect(a.table().count > 20)249 // Codable round trip250 let data = try JSONEncoder().encode(a)251 #expect(try JSONDecoder().decode(MonteCarloResult.self, from: data) == a)252 }253254 @Test func multiOutputProgressAndCancellation() throws {255 var opt = MonteCarloOptions(trials: 2000, seed: 5)256 opt.progressInterval = 100257 var ticks = [Int]()258 let full = try MonteCarlo.run(inputs: inputs, outputNames: ["revenue", "margin"], options: opt, progress: { d, t in ticks.append(d); #expect(t == 2000) }) {259 [$0[0] * $0[1], $0[0] - $0[2]]260 }261 #expect(full.outputs.count == 2 && full.outputs[0].name == "revenue" && full.outputs[1].name == "margin")262 #expect(ticks.first == 0 && ticks.last == 2000 && ticks.count == 21)263 #expect(isClose(full.outputs[1].mean, Descriptive.mean(zip(full.inputSamples[0], full.inputSamples[2]).map { $0 - $1 }), rel: 1e-12))264 var polls = 0265 let part = try MonteCarlo.run(inputs: inputs, outputNames: ["revenue", "margin"], options: opt, isCancelled: { polls += 1; return polls > 5 }) {266 [$0[0] * $0[1], $0[0] - $0[2]]267 }268 #expect(part.cancelled && part.trialsCompleted == 500 && part.trialsRequested == 2000)269 #expect(part.outputs[0].count == 500 && part.inputSamples[0].count == 500 && part.outputSamples[1].count == 500)270 // the completed prefix of a cancelled run is identical to the full run271 #expect(part.outputSamples[0] == Array(full.outputSamples[0].prefix(500)))272 #expect(throws: StatError.self) { try MonteCarlo.run(inputs: [], outputName: "x") { _ in 0 } }273 #expect(throws: StatError.self) { try MonteCarlo.run(inputs: self.inputs, outputNames: ["a", "b"]) { _ in [1] } }274 #expect(throws: StatError.self) { try MonteCarlo.run(inputs: [MonteCarloInput(name: "bad", distribution: .uniform(min: 2, max: 1))], outputName: "x") { $0[0] } }275 var zero = MonteCarloOptions(); zero.trials = 0276 #expect(throws: StatError.self) { try MonteCarlo.run(inputs: self.inputs, outputName: "x", options: zero) { $0[0] } }277 }278279 let target: [[Double]] = [[1, 0.7, -0.4], [0.7, 1, 0.2], [-0.4, 0.2, 1]]280281 @Test func imanConoverPreservesMarginalsAndHitsRankCorrelation() throws {282 let n = 20_000283 let independent = try MonteCarlo.sampleInputs(inputs, trials: n, seed: 42)284 let correlated = try MonteCarlo.sampleInputs(inputs, trials: n, seed: 42, correlation: MonteCarloCorrelation(matrix: target, method: .imanConover))285 for j in 0..<3 {286 #expect(independent[j].sorted() == correlated[j].sorted(), "marginal \(j) is a permutation of the independent draw")287 #expect(independent[j] != correlated[j])288 }289 let achieved = ResamplingMath.spearmanMatrix(correlated)290 for i in 0..<3 { for j in 0..<3 { #expect(abs(achieved[i][j] - target[i][j]) < 0.02, "ρ[\(i)][\(j)] = \(achieved[i][j]) vs \(target[i][j])") } }291 let base = ResamplingMath.spearmanMatrix(independent)292 #expect(abs(base[0][1]) < 0.03 && abs(base[0][2]) < 0.03 && abs(base[1][2]) < 0.03)293 #expect(try MonteCarlo.sampleInputs(inputs, trials: n, seed: 42, correlation: MonteCarloCorrelation(matrix: target)) == correlated)294 }295296 @Test func gaussianCopulaHitsRankCorrelationAndMarginals() throws {297 let n = 20_000298 let cols = try MonteCarlo.sampleInputs(inputs, trials: n, seed: 9, correlation: MonteCarloCorrelation(matrix: target, method: .gaussianCopula))299 let achieved = ResamplingMath.spearmanMatrix(cols)300 for i in 0..<3 { for j in 0..<3 { #expect(abs(achieved[i][j] - target[i][j]) < 0.02, "ρ[\(i)][\(j)] = \(achieved[i][j])") } }301 for (j, i) in inputs.enumerated() {302 let d = i.distribution303 #expect(abs(Descriptive.mean(cols[j]) - d.mean) < 5 * (d.variance / Double(n)).squareRoot(), "\(i.name) mean")304 #expect(isClose(Descriptive.variance(cols[j]), d.variance, rel: 0.06), "\(i.name) variance")305 }306 #expect(cols[0].allSatisfy { $0 >= 8 && $0 <= 13 })307 // the Spearman→Pearson conversion used for the normal scores308 for pair in f["spearman_to_pearson"].list { #expect(isClose(2 * sin(Double.pi * pair[0].d / 6), pair[1].d, rel: 1e-14, abs: 1e-16)) }309 // a perfectly comonotone pair310 let pair = Array(inputs.prefix(2))311 let mono = try MonteCarlo.sampleInputs(pair, trials: 2000, seed: 1, correlation: MonteCarloCorrelation(matrix: [[1, 0.999], [0.999, 1]], method: .gaussianCopula))312 #expect(ResamplingMath.spearman(mono[0], mono[1]) > 0.995)313 }314315 @Test func correlationFlowsThroughRunAndBadMatricesAreRejected() throws {316 var opt = MonteCarloOptions(trials: 8000, seed: 3)317 opt.correlation = MonteCarloCorrelation(matrix: target, method: .imanConover)318 let r = try MonteCarlo.run(inputs: inputs, outputName: "profit", options: opt) { ($0[0] - $0[2]) * $0[1] }319 let a = try #require(r.achievedInputRankCorrelation)320 #expect(abs(a[0][1] - 0.7) < 0.03 && abs(a[0][2] + 0.4) < 0.03 && abs(a[1][2] - 0.2) < 0.03)321 opt.correlation = nil322 let ind = try MonteCarlo.run(inputs: inputs, outputName: "profit", options: opt) { ($0[0] - $0[2]) * $0[1] }323 // positive price–volume dependence widens the profit distribution324 #expect(r.outputs[0].standardDeviation > ind.outputs[0].standardDeviation)325 let bad: [[[Double]]] = [326 [[1, 0.5], [0.5, 1]], // wrong size327 [[1, 0.9, -0.9], [0.9, 1, 0.9], [-0.9, 0.9, 1]], // not positive definite328 [[1, 0.5, 0], [0.4, 1, 0], [0, 0, 1]], // asymmetric329 [[2, 0, 0], [0, 1, 0], [0, 0, 1]], // diagonal330 [[1, 1.5, 0], [1.5, 1, 0], [0, 0, 1]], // out of range331 ]332 for m in bad {333 for method in MonteCarloCorrelation.Method.allCases {334 #expect(throws: StatError.self) { try MonteCarlo.sampleInputs(self.inputs, trials: 100, seed: 1, correlation: MonteCarloCorrelation(matrix: m, method: method)) }335 }336 }337 }338}339