Repository Public MIT
1// ORIX Calc2// File: Packages/ORIXFeatureStatistics/Tests/ORIXFeatureStatisticsTests/InferenceTests.swift3// Description: Tests of the Inference section — every procedure from data and from summary statistics against the reference values, decision wording, staleness, rejection regions.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Testing9import ORIXExpression10import ORIXStatistics11import ORIXUI12@testable import ORIXFeatureStatistics1314@MainActor @Suite struct InferenceTests {15 private func alternative(_ s: String) -> Alternative { Alternative(rawValue: s) ?? .twoSided }16 private func dataset(_ columns: [Double]...) -> Dataset {17 Dataset(title: "ref", columns: columns.enumerated().map { DataColumn(name: Dataset.defaultNames[$0.offset], values: $0.element.map(Optional.some)) })18 }19 private func summaryOf(_ x: [Double]) -> (n: Double, mean: Double, sd: Double) {20 let n = Double(x.count), m = x.reduce(0, +) / n21 return (n, m, (x.map { ($0 - m) * ($0 - m) }.reduce(0, +) / (n - 1)).squareRoot())22 }2324 @Test func demoTTestMatchesAnIndependentCalculation() throws {25 let store = StatisticsStore()26 store.data.loadDemo()27 store.inference.loadDemo(dataset: store.data.dataset, dataRevision: store.data.revision)28 let o = try #require(store.inference.output)29 let t = try #require(o.test)30 let y: [Double] = [52, 58, 57, 64, 68, 66, 72, 79, 77, 83, 86, 93]31 let s = summaryOf(y)32 #expect(close(t.statistic, (s.mean - 65) / (s.sd / s.n.squareRoot()), 1e-12))33 #expect(t.degreesOfFreedom == [11])34 #expect(t.alternative == .greater)35 // One-sided critical value t(0.95; 11) from tables: 1.79588536 #expect(abs(t.criticalValue - 1.795885) < 1e-6)37 #expect(t.rejectNull == (t.statistic >= t.criticalValue))38 #expect(t.rejectNull == (t.pValue <= 0.05))39 let f = StatFormat()40 #expect(o.decision(f) == "\(t.rejectNull ? "Reject" : "Fail to reject") H₀ at α = 0.05")41 #expect(store.inference.error == nil)42 #expect(store.inference.state.calculated)43 #expect(o.reference?.symbol == "t")44 #expect(!o.assumptions.isEmpty)45 }4647 @Test func decisionWordingNeverAccepts() throws {48 var s = InferenceState()49 s.procedure = .tTest; s.source = .summary; s.n1 = 25; s.mean1 = 52.3; s.sd1 = 6.1; s.mu0 = 5250 let f = StatFormat()51 let notRejected = try InferenceEngine.compute(s, dataset: Dataset())52 #expect(notRejected.decision(f) == "Fail to reject H₀ at α = 0.05")53 #expect(notRejected.decisionDetail(f)?.contains("does not show that H₀ is true") == true)54 s.mu0 = 4055 let rejected = try InferenceEngine.compute(s, dataset: Dataset())56 #expect(rejected.decision(f) == "Reject H₀ at α = 0.05")57 #expect(rejected.decisionDetail(f)?.contains("p < 0.0001") == true)58 for text in [notRejected.decision(f), notRejected.decisionDetail(f), rejected.decision(f), rejected.decisionDetail(f)] {59 #expect(text?.lowercased().contains("accept") == false)60 }61 }6263 @Test func pValueFormatting() {64 let f = StatFormat()65 #expect(f.pValue(0.00009999) == "< 0.0001")66 #expect(f.pValue(0.0001) == "0.0001")67 #expect(f.pValue(0.000123456) == "0.0001235")68 #expect(f.pValue(0.0243341165) == "0.02433")69 #expect(f.pValue(0.5) == "0.5")70 #expect(f.pValue(1) == "1")71 #expect(f.pValue(3e-9) == "< 0.0001")72 var comma = NumberFormat(); comma.decimalSeparator = ","73 #expect(StatFormat(comma).pValue(1e-7) == "< 0,0001")74 }7576 @Test func oneSampleTestsFromDataAndSummary() throws {77 for c in try Reference.cases("inference") where c.string("kind") == "oneSampleT" || c.string("kind") == "oneSampleZ" {78 let data = c.doubles("data"), isT = c.string("kind") == "oneSampleT"79 var s = InferenceState()80 s.procedure = isT ? .tTest : .zTest81 s.alpha = c.double("alpha"); s.alternative = alternative(c.string("alternative")); s.mu0 = c.double("mu0")82 if !isT { s.sigma = c.double("sigma") }83 s.source = .data; s.columnA = 084 let fromData = try #require(try InferenceEngine.compute(s, dataset: dataset(data)).test)85 let sum = summaryOf(data)86 s.source = .summary; s.n1 = sum.n; s.mean1 = sum.mean; s.sd1 = sum.sd87 let fromSummary = try #require(try InferenceEngine.compute(s, dataset: Dataset()).test)88 for t in [fromData, fromSummary] {89 #expect(close(t.statistic, c.double("statistic"), 1e-9))90 #expect(close(t.pValue, c.double("p"), 1e-8))91 #expect(close(abs(t.criticalValue), abs(c.double("critical")), 1e-9) || s.alternative != .twoSided)92 let ci = try #require(t.confidenceInterval)93 #expect(close(ci.lower, c.double("ciLower"), 1e-9) && close(ci.upper, c.double("ciUpper"), 1e-9))94 if isT { #expect(t.degreesOfFreedom == [c.double("df")]) }95 if c.has("cohenD") { #expect(close(try #require(t.effectSize).value, c.double("cohenD"), 1e-9)) }96 }97 // The matching interval procedure gives the same limits at level 1 − α.98 var ci = InferenceState()99 ci.procedure = isT ? .ciMeanT : .ciMeanZ; ci.source = .data; ci.level = 1 - c.double("alpha"); ci.sigma = s.sigma100 let interval = try #require(try InferenceEngine.compute(ci, dataset: dataset(data)).interval)101 #expect(close(interval.lower, c.double("ciLower"), 1e-9) && close(interval.upper, c.double("ciUpper"), 1e-9))102 }103 }104105 @Test func twoSampleAndPairedTests() throws {106 for c in try Reference.cases("inference") where c.string("kind") == "twoSampleT" {107 var s = InferenceState()108 s.procedure = .twoSampleT; s.source = .data; s.columnA = 0; s.columnB = 1109 s.alpha = c.double("alpha"); s.alternative = alternative(c.string("alternative")); s.twoSample = c.bool("pooled") ? .pooled : .welch110 let ds = dataset(c.doubles("a"), c.doubles("b"))111 #expect(ds.rowCount == max(c.doubles("a").count, c.doubles("b").count), "columns of different lengths are padded with empty cells")112 let t = try #require(try InferenceEngine.compute(s, dataset: ds).test)113 #expect(close(t.statistic, c.double("statistic"), 1e-9))114 #expect(close(t.pValue, c.double("p"), 1e-8))115 #expect(close(t.degreesOfFreedom[0], c.double("df"), 1e-9))116 let sum = c.doubles("summary")117 s.source = .summary; s.n1 = sum[0]; s.mean1 = sum[1]; s.sd1 = sum[2]; s.n2 = sum[3]; s.mean2 = sum[4]; s.sd2 = sum[5]118 let fromSummary = try #require(try InferenceEngine.compute(s, dataset: Dataset()).test)119 #expect(close(fromSummary.statistic, c.double("statistic"), 1e-9))120 var ci = s121 ci.procedure = .ciTwoMeans; ci.level = 1 - c.double("alpha")122 let interval = try #require(try InferenceEngine.compute(ci, dataset: Dataset()).interval)123 #expect(close(interval.lower, c.double("ciLower"), 1e-8) && close(interval.upper, c.double("ciUpper"), 1e-8))124 }125 for c in try Reference.cases("inference") where c.string("kind") == "pairedT" {126 let a = c.doubles("a"), b = c.doubles("b")127 var s = InferenceState()128 s.procedure = .pairedT; s.source = .data; s.columnA = 0; s.columnB = 1; s.alternative = alternative(c.string("alternative"))129 let t = try #require(try InferenceEngine.compute(s, dataset: dataset(a, b)).test)130 #expect(close(t.statistic, c.double("statistic"), 1e-9))131 #expect(close(t.pValue, c.double("p"), 1e-8))132 // From the summary of the differences.133 let d = summaryOf(zip(a, b).map { $0 - $1 })134 s.source = .summary; s.n1 = d.n; s.mean1 = d.mean; s.sd1 = d.sd135 let o = try InferenceEngine.compute(s, dataset: Dataset())136 #expect(close(try #require(o.test).statistic, c.double("statistic"), 1e-9))137 #expect(!o.notes.isEmpty)138 var ci = InferenceState()139 ci.procedure = .ciPaired; ci.source = .data140 if s.alternative == .twoSided {141 let interval = try #require(try InferenceEngine.compute(ci, dataset: dataset(a, b)).interval)142 #expect(close(interval.lower, c.double("ciLower"), 1e-9) && close(interval.upper, c.double("ciUpper"), 1e-9))143 }144 }145 }146147 @Test func proportions() throws {148 for c in try Reference.cases("inference") where c.string("kind") == "proportionCI" {149 var s = InferenceState()150 s.procedure = .ciProportion; s.source = .summary; s.x1 = c.double("x"); s.trials1 = c.double("n"); s.level = c.double("level")151 for (method, key) in [(ProportionIntervalMethod.wilson, "wilson"), (.wald, "wald"), (.agrestiCoull, "agrestiCoull"), (.clopperPearson, "exact")] {152 s.proportionMethod = method153 let ci = try #require(try InferenceEngine.compute(s, dataset: Dataset()).interval)154 #expect(close(ci.lower, c.doubles(key)[0], 1e-8) && close(ci.upper, c.doubles(key)[1], 1e-8), "\(key)")155 }156 // The same counts from a 0/1 column.157 let x = c.int("x"), n = c.int("n")158 s.source = .data; s.proportionMethod = .wilson159 let ci = try #require(try InferenceEngine.compute(s, dataset: dataset(Array(repeating: 1, count: x) + Array(repeating: 0, count: n - x))).interval)160 #expect(close(ci.lower, c.doubles("wilson")[0], 1e-8))161 }162 #expect(InferenceState().proportionMethod == .wilson, "Wilson is the default")163 for c in try Reference.cases("inference") where c.string("kind") == "oneProportionZ" {164 var s = InferenceState()165 s.procedure = .onePropZ; s.source = .summary; s.x1 = c.double("x"); s.trials1 = c.double("n"); s.p0 = c.double("p0"); s.alternative = alternative(c.string("alternative"))166 let t = try #require(try InferenceEngine.compute(s, dataset: Dataset()).test)167 #expect(close(t.statistic, c.double("statistic"), 1e-9) && close(t.pValue, c.double("p"), 1e-8))168 s.continuity = true169 let cc = try #require(try InferenceEngine.compute(s, dataset: Dataset()).test)170 #expect(close(cc.statistic, c.double("statisticCC"), 1e-9) && close(cc.pValue, c.double("pCC"), 1e-8))171 }172 for c in try Reference.cases("inference") where c.string("kind") == "twoProportionZ" {173 var s = InferenceState()174 s.procedure = .twoPropZ; s.source = .summary; s.x1 = c.double("x1"); s.trials1 = c.double("n1"); s.x2 = c.double("x2"); s.trials2 = c.double("n2")175 let t = try #require(try InferenceEngine.compute(s, dataset: Dataset()).test)176 #expect(close(t.statistic, c.double("statistic"), 1e-9) && close(t.pValue, c.double("p"), 1e-8))177 s.alternative = .greater178 #expect(close(try #require(try InferenceEngine.compute(s, dataset: Dataset()).test).pValue, c.double("pGreater"), 1e-8))179 s.procedure = .ciTwoProportions180 let ci = try #require(try InferenceEngine.compute(s, dataset: Dataset()).interval)181 #expect(close(ci.lower, c.double("ciLower"), 1e-8) && close(ci.upper, c.double("ciUpper"), 1e-8))182 }183 var bad = InferenceState()184 bad.procedure = .onePropZ; bad.source = .data185 #expect(throws: InputError.self) { try InferenceEngine.compute(bad, dataset: self.dataset([0, 1, 2])) }186 }187188 @Test func chiSquareTests() throws {189 for c in try Reference.cases("inference") where c.string("kind") == "gof" {190 var s = InferenceState()191 s.procedure = .gof; s.source = .summary192 s.gofObserved = c.doubles("observed").map { String($0) }.joined(separator: " ")193 if c.has("proportions") {194 s.expectedKind = .proportions; s.gofExpected = c.doubles("proportions").map { String($0) }.joined(separator: ", ")195 }196 guard c.int("ddof") == 0 else { continue }197 let o = try InferenceEngine.compute(s, dataset: Dataset())198 let t = try #require(o.test)199 #expect(close(t.statistic, c.double("statistic"), 1e-9) && close(t.pValue, c.double("p"), 1e-8))200 #expect(close(t.criticalValue, c.double("critical"), 1e-9))201 #expect(zip(try #require(o.gof).expected, c.doubles("expected")).allSatisfy { close($0, $1, 1e-9) })202 // Counts given as expected counts, from columns.203 s.source = .data; s.expectedKind = .counts; s.columnA = 0; s.columnB = 1204 let fromColumns = try #require(try InferenceEngine.compute(s, dataset: dataset(c.doubles("observed"), c.doubles("expected"))).test)205 #expect(close(fromColumns.statistic, c.double("statistic"), 1e-9))206 }207 for c in try Reference.cases("inference") where c.string("kind") == "independence" {208 var s = InferenceState()209 s.procedure = .independence; s.source = .summary; s.yates = c.bool("yates")210 s.table = c.matrix("table").map { $0.map(Optional.some) }211 let o = try InferenceEngine.compute(s, dataset: Dataset())212 let r = try #require(o.independence)213 #expect(close(r.test.statistic, c.double("statistic"), 1e-9) && close(r.test.pValue, c.double("p"), 1e-8))214 #expect(close(r.cramersV, c.double("cramersV"), 1e-9))215 #expect(zip(r.expected.joined(), c.matrix("expected").joined()).allSatisfy { close($0, $1, 1e-9) })216 #expect(r.yatesApplied == c.bool("yatesApplied"))217 }218 // An empty cell is reported, not treated as zero.219 var s = InferenceState()220 s.procedure = .independence; s.source = .summary; s.table = [[12, nil], [7, 14]]221 #expect(throws: InputError.self) { try InferenceEngine.compute(s, dataset: Dataset()) }222 // Small expected counts carry the engine's warning.223 s.table = [[3, 2], [1, 4]]224 let small = try InferenceEngine.compute(s, dataset: Dataset())225 #expect(try #require(small.independence).expected.joined().contains { $0 < 5 })226 #expect(!small.verification.warnings.isEmpty)227 // Typed lists report what cannot be read.228 s.procedure = .gof; s.gofObserved = "18 22 x 15"229 #expect(throws: InputError.self) { try InferenceEngine.compute(s, dataset: Dataset()) }230 }231232 @Test func independenceFromTwoCategoryColumns() throws {233 // Rebuild the 2 × 2 table [[12, 5], [7, 14]] as 38 observations.234 var x: [Double] = [], y: [Double] = []235 for (r, row) in [[12, 5], [7, 14]].enumerated() { for (c, count) in row.enumerated() { for _ in 0..<count { x.append(Double(r)); y.append(Double(c + 10)) } } }236 #expect(try InferenceEngine.crossTab(x, y).table == [[12, 5], [7, 14]])237 var s = InferenceState()238 s.procedure = .independence; s.source = .data; s.columnA = 0; s.columnB = 1239 let r = try #require(try InferenceEngine.compute(s, dataset: dataset(x, y)).independence)240 #expect(close(r.test.statistic, 5.215686274509804, 1e-12))241 #expect(throws: InputError.self) { try InferenceEngine.crossTab([1, 1, 1], [1, 2, 3]) }242 }243244 @Test func anovaFTestCorrelationVariance() throws {245 for c in try Reference.cases("inference") where c.string("kind") == "anova" {246 let groups = c.matrix("groups")247 var s = InferenceState()248 s.procedure = .anova; s.source = .data; s.groupColumns = Array(groups.indices)249 let ds = Dataset(title: "g", columns: groups.enumerated().map { DataColumn(name: "G\($0.offset + 1)", values: $0.element.map(Optional.some)) })250 let a = try #require(try InferenceEngine.compute(s, dataset: ds).anova)251 #expect(close(a.test.statistic, c.double("statistic"), 1e-9) && close(a.test.pValue, c.double("p"), 1e-8))252 #expect(close(a.sumOfSquaresBetween, c.double("ssb"), 1e-9) && close(a.sumOfSquaresWithin, c.double("ssw"), 1e-9))253 #expect(close(a.meanSquareBetween, c.double("msb"), 1e-9) && close(a.meanSquareWithin, c.double("msw"), 1e-9))254 #expect(a.degreesOfFreedomBetween == c.double("df1") && a.degreesOfFreedomWithin == c.double("df2"))255 #expect(close(a.etaSquared, c.double("etaSquared"), 1e-9))256 s.source = .summary257 s.groups = c.matrix("summaries").map { GroupSummaryInput(n: $0[0], mean: $0[1], sd: $0[2]) }258 let fromSummary = try #require(try InferenceEngine.compute(s, dataset: Dataset()).anova)259 #expect(close(fromSummary.test.statistic, c.double("statistic"), 1e-8))260 s.source = .data; s.groupColumns = [0]261 #expect(throws: InputError.self) { try InferenceEngine.compute(s, dataset: ds) }262 }263 for c in try Reference.cases("inference") where c.string("kind") == "fTest" {264 var s = InferenceState()265 s.procedure = .fTest; s.source = .data; s.columnA = 0; s.columnB = 1266 let t = try #require(try InferenceEngine.compute(s, dataset: dataset(c.doubles("a"), c.doubles("b"))).test)267 #expect(close(t.statistic, c.double("statistic"), 1e-9) && close(t.pValue, c.double("p"), 1e-8))268 #expect(close(try #require(t.lowerCriticalValue), c.double("criticalLower"), 1e-9) && close(t.criticalValue, c.double("criticalUpper"), 1e-9))269 let a = summaryOf(c.doubles("a")), b = summaryOf(c.doubles("b"))270 s.source = .summary; s.n1 = a.n; s.sd1 = a.sd; s.n2 = b.n; s.sd2 = b.sd; s.alternative = .greater271 #expect(close(try #require(try InferenceEngine.compute(s, dataset: Dataset()).test).pValue, c.double("pGreater"), 1e-8))272 }273 for c in try Reference.cases("inference") where c.string("kind") == "correlation" {274 var s = InferenceState()275 s.procedure = .correlation; s.source = .data; s.columnA = 0; s.columnB = 1276 let t = try #require(try InferenceEngine.compute(s, dataset: dataset(c.doubles("x"), c.doubles("y"))).test)277 #expect(close(t.statistic, c.double("statistic"), 1e-9) && close(t.pValue, c.double("p"), 1e-8))278 s.source = .summary; s.r = c.double("r"); s.nPairs = Double(c.doubles("x").count); s.alternative = .less279 #expect(close(try #require(try InferenceEngine.compute(s, dataset: Dataset()).test).pValue, c.double("pLess"), 1e-8))280 }281 for c in try Reference.cases("inference") where c.string("kind") == "varianceCI" {282 var s = InferenceState()283 s.procedure = .ciVariance; s.source = .data; s.level = c.double("level")284 let o = try InferenceEngine.compute(s, dataset: dataset(c.doubles("data")))285 let ci = try #require(o.interval)286 #expect(close(ci.lower, c.double("lower"), 1e-8) && close(ci.upper, c.double("upper"), 1e-8))287 let sd = try #require(o.secondaryInterval)288 #expect(close(sd.lower, c.double("lower").squareRoot(), 1e-8))289 let sum = summaryOf(c.doubles("data"))290 s.source = .summary; s.n1 = sum.n; s.sd1 = sum.sd291 #expect(close(try #require(try InferenceEngine.compute(s, dataset: Dataset()).interval).upper, c.double("upper"), 1e-8))292 }293 }294295 @Test func everyProcedureComputesWithItsDefaults() throws {296 let ds = SampleDatasets.bloodPressure297 for p in InferenceProcedure.allCases {298 var s = InferenceState()299 s.procedure = p; s.source = .summary300 let o = try InferenceEngine.compute(s, dataset: ds)301 #expect(o.interval != nil || o.test != nil, "\(p)")302 #expect(o.procedure.isInterval == (o.test == nil))303 if let t = o.test {304 let plot = try #require(o.reference, "\(p)")305 #expect(plot.points.count > 100 && plot.points.contains { $0.rejected })306 // The rejection region has probability α under the reference distribution.307 let d = try #require(InferenceEngine.referenceDistribution(t))308 let mass = InferenceEngine.rejectionRegion(t).map { d.cdf($0.upperBound) - d.cdf($0.lowerBound) }.reduce(0, +)309 #expect(abs(mass - t.alpha) < 1e-9, "\(p): region mass \(mass)")310 // The statistic is in the region exactly when H₀ is rejected.311 #expect(InferenceEngine.rejectionRegion(t).contains { $0.contains(t.statistic) } == t.rejectNull, "\(p)")312 }313 }314 #expect(Set(InferenceProcedure.intervals + InferenceProcedure.tests) == Set(InferenceProcedure.allCases))315 }316317 @Test func inputErrorsAreExplained() {318 let m = InferenceModel()319 m.state.procedure = .tTest; m.state.source = .summary; m.state.n1 = 12.5320 #expect(m.calculate(dataset: Dataset(), dataRevision: 0) == nil)321 #expect(m.error?.contains("whole number") == true)322 #expect(!m.state.calculated)323 m.state.n1 = 12; m.state.sd1 = nil324 m.calculate(dataset: Dataset(), dataRevision: 0)325 #expect(m.error?.contains("standard deviation") == true)326 m.state.sd1 = 3; m.state.alpha = 1.5327 m.calculate(dataset: Dataset(), dataRevision: 0)328 #expect(m.error == StatisticsError.probabilityOutOfRange(1.5).description, "the engine's own message")329 m.state.alpha = 0.05; m.state.source = .data; m.state.columnA = 0330 m.calculate(dataset: Dataset(), dataRevision: 0)331 #expect(m.error?.contains("no values") == true)332 }333334 @Test func staleResultsAreFlagged() {335 let data = DataModel()336 data.loadDemo()337 let m = InferenceModel()338 m.loadDemo(dataset: data.dataset, dataRevision: data.revision)339 #expect(!m.isStale(dataRevision: data.revision))340 m.state.alpha = 0.01341 #expect(m.isStale(dataRevision: data.revision))342 m.calculate(dataset: data.dataset, dataRevision: data.revision)343 #expect(!m.isStale(dataRevision: data.revision))344 data.setValue(99, row: 0, column: 1)345 #expect(m.isStale(dataRevision: data.revision), "the dataset changed")346 }347348 @Test func calculateLogsToTheSharedHistory() throws {349 let app = makeApp()350 let store = StatisticsStore()351 store.data.loadDemo()352 store.inference.state.procedure = .tTest353 store.inference.state.source = .data; store.inference.state.columnA = 1; store.inference.state.mu0 = 65354 store.calculateInference(app)355 let entry = try #require(app.history.first)356 #expect(entry.mode == .statistics)357 #expect(entry.input.contains("t test") && entry.display.contains("H₀"))358 #expect(!entry.detail.isEmpty, "the explanation is stored")359 store.inference.state.procedure = .ciProportion; store.inference.state.source = .summary360 store.calculateInference(app)361 #expect(app.history.count == 2 && app.history.first?.display.contains("CI") == true)362 store.inference.state.trials1 = nil363 store.calculateInference(app)364 #expect(app.history.count == 2, "a failed calculation is not logged")365 }366}367