SPB Git

Repository Public MIT

InferenceTests.swift 22.8 KB 367 lines swift

1// ORIX Calc2// File: Packages/ORIXFeatureStatistics/Tests/ORIXFeatureStatisticsTests/InferenceTests.swift3// Description: Tests of the Inference section — every procedure from data and from summary statistics against the reference values, decision wording, staleness, rejection regions.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Testing9import ORIXExpression10import ORIXStatistics11import ORIXUI12@testable import ORIXFeatureStatistics1314@MainActor @Suite struct InferenceTests {15    private func alternative(_ s: String) -> Alternative { Alternative(rawValue: s) ?? .twoSided }16    private func dataset(_ columns: [Double]...) -> Dataset {17        Dataset(title: "ref", columns: columns.enumerated().map { DataColumn(name: Dataset.defaultNames[$0.offset], values: $0.element.map(Optional.some)) })18    }19    private func summaryOf(_ x: [Double]) -> (n: Double, mean: Double, sd: Double) {20        let n = Double(x.count), m = x.reduce(0, +) / n21        return (n, m, (x.map { ($0 - m) * ($0 - m) }.reduce(0, +) / (n - 1)).squareRoot())22    }2324    @Test func demoTTestMatchesAnIndependentCalculation() throws {25        let store = StatisticsStore()26        store.data.loadDemo()27        store.inference.loadDemo(dataset: store.data.dataset, dataRevision: store.data.revision)28        let o = try #require(store.inference.output)29        let t = try #require(o.test)30        let y: [Double] = [52, 58, 57, 64, 68, 66, 72, 79, 77, 83, 86, 93]31        let s = summaryOf(y)32        #expect(close(t.statistic, (s.mean - 65) / (s.sd / s.n.squareRoot()), 1e-12))33        #expect(t.degreesOfFreedom == [11])34        #expect(t.alternative == .greater)35        // One-sided critical value t(0.95; 11) from tables: 1.79588536        #expect(abs(t.criticalValue - 1.795885) < 1e-6)37        #expect(t.rejectNull == (t.statistic >= t.criticalValue))38        #expect(t.rejectNull == (t.pValue <= 0.05))39        let f = StatFormat()40        #expect(o.decision(f) == "\(t.rejectNull ? "Reject" : "Fail to reject") H₀ at α = 0.05")41        #expect(store.inference.error == nil)42        #expect(store.inference.state.calculated)43        #expect(o.reference?.symbol == "t")44        #expect(!o.assumptions.isEmpty)45    }4647    @Test func decisionWordingNeverAccepts() throws {48        var s = InferenceState()49        s.procedure = .tTest; s.source = .summary; s.n1 = 25; s.mean1 = 52.3; s.sd1 = 6.1; s.mu0 = 5250        let f = StatFormat()51        let notRejected = try InferenceEngine.compute(s, dataset: Dataset())52        #expect(notRejected.decision(f) == "Fail to reject H₀ at α = 0.05")53        #expect(notRejected.decisionDetail(f)?.contains("does not show that H₀ is true") == true)54        s.mu0 = 4055        let rejected = try InferenceEngine.compute(s, dataset: Dataset())56        #expect(rejected.decision(f) == "Reject H₀ at α = 0.05")57        #expect(rejected.decisionDetail(f)?.contains("p < 0.0001") == true)58        for text in [notRejected.decision(f), notRejected.decisionDetail(f), rejected.decision(f), rejected.decisionDetail(f)] {59            #expect(text?.lowercased().contains("accept") == false)60        }61    }6263    @Test func pValueFormatting() {64        let f = StatFormat()65        #expect(f.pValue(0.00009999) == "< 0.0001")66        #expect(f.pValue(0.0001) == "0.0001")67        #expect(f.pValue(0.000123456) == "0.0001235")68        #expect(f.pValue(0.0243341165) == "0.02433")69        #expect(f.pValue(0.5) == "0.5")70        #expect(f.pValue(1) == "1")71        #expect(f.pValue(3e-9) == "< 0.0001")72        var comma = NumberFormat(); comma.decimalSeparator = ","73        #expect(StatFormat(comma).pValue(1e-7) == "< 0,0001")74    }7576    @Test func oneSampleTestsFromDataAndSummary() throws {77        for c in try Reference.cases("inference") where c.string("kind") == "oneSampleT" || c.string("kind") == "oneSampleZ" {78            let data = c.doubles("data"), isT = c.string("kind") == "oneSampleT"79            var s = InferenceState()80            s.procedure = isT ? .tTest : .zTest81            s.alpha = c.double("alpha"); s.alternative = alternative(c.string("alternative")); s.mu0 = c.double("mu0")82            if !isT { s.sigma = c.double("sigma") }83            s.source = .data; s.columnA = 084            let fromData = try #require(try InferenceEngine.compute(s, dataset: dataset(data)).test)85            let sum = summaryOf(data)86            s.source = .summary; s.n1 = sum.n; s.mean1 = sum.mean; s.sd1 = sum.sd87            let fromSummary = try #require(try InferenceEngine.compute(s, dataset: Dataset()).test)88            for t in [fromData, fromSummary] {89                #expect(close(t.statistic, c.double("statistic"), 1e-9))90                #expect(close(t.pValue, c.double("p"), 1e-8))91                #expect(close(abs(t.criticalValue), abs(c.double("critical")), 1e-9) || s.alternative != .twoSided)92                let ci = try #require(t.confidenceInterval)93                #expect(close(ci.lower, c.double("ciLower"), 1e-9) && close(ci.upper, c.double("ciUpper"), 1e-9))94                if isT { #expect(t.degreesOfFreedom == [c.double("df")]) }95                if c.has("cohenD") { #expect(close(try #require(t.effectSize).value, c.double("cohenD"), 1e-9)) }96            }97            // The matching interval procedure gives the same limits at level 1 − α.98            var ci = InferenceState()99            ci.procedure = isT ? .ciMeanT : .ciMeanZ; ci.source = .data; ci.level = 1 - c.double("alpha"); ci.sigma = s.sigma100            let interval = try #require(try InferenceEngine.compute(ci, dataset: dataset(data)).interval)101            #expect(close(interval.lower, c.double("ciLower"), 1e-9) && close(interval.upper, c.double("ciUpper"), 1e-9))102        }103    }104105    @Test func twoSampleAndPairedTests() throws {106        for c in try Reference.cases("inference") where c.string("kind") == "twoSampleT" {107            var s = InferenceState()108            s.procedure = .twoSampleT; s.source = .data; s.columnA = 0; s.columnB = 1109            s.alpha = c.double("alpha"); s.alternative = alternative(c.string("alternative")); s.twoSample = c.bool("pooled") ? .pooled : .welch110            let ds = dataset(c.doubles("a"), c.doubles("b"))111            #expect(ds.rowCount == max(c.doubles("a").count, c.doubles("b").count), "columns of different lengths are padded with empty cells")112            let t = try #require(try InferenceEngine.compute(s, dataset: ds).test)113            #expect(close(t.statistic, c.double("statistic"), 1e-9))114            #expect(close(t.pValue, c.double("p"), 1e-8))115            #expect(close(t.degreesOfFreedom[0], c.double("df"), 1e-9))116            let sum = c.doubles("summary")117            s.source = .summary; s.n1 = sum[0]; s.mean1 = sum[1]; s.sd1 = sum[2]; s.n2 = sum[3]; s.mean2 = sum[4]; s.sd2 = sum[5]118            let fromSummary = try #require(try InferenceEngine.compute(s, dataset: Dataset()).test)119            #expect(close(fromSummary.statistic, c.double("statistic"), 1e-9))120            var ci = s121            ci.procedure = .ciTwoMeans; ci.level = 1 - c.double("alpha")122            let interval = try #require(try InferenceEngine.compute(ci, dataset: Dataset()).interval)123            #expect(close(interval.lower, c.double("ciLower"), 1e-8) && close(interval.upper, c.double("ciUpper"), 1e-8))124        }125        for c in try Reference.cases("inference") where c.string("kind") == "pairedT" {126            let a = c.doubles("a"), b = c.doubles("b")127            var s = InferenceState()128            s.procedure = .pairedT; s.source = .data; s.columnA = 0; s.columnB = 1; s.alternative = alternative(c.string("alternative"))129            let t = try #require(try InferenceEngine.compute(s, dataset: dataset(a, b)).test)130            #expect(close(t.statistic, c.double("statistic"), 1e-9))131            #expect(close(t.pValue, c.double("p"), 1e-8))132            // From the summary of the differences.133            let d = summaryOf(zip(a, b).map { $0 - $1 })134            s.source = .summary; s.n1 = d.n; s.mean1 = d.mean; s.sd1 = d.sd135            let o = try InferenceEngine.compute(s, dataset: Dataset())136            #expect(close(try #require(o.test).statistic, c.double("statistic"), 1e-9))137            #expect(!o.notes.isEmpty)138            var ci = InferenceState()139            ci.procedure = .ciPaired; ci.source = .data140            if s.alternative == .twoSided {141                let interval = try #require(try InferenceEngine.compute(ci, dataset: dataset(a, b)).interval)142                #expect(close(interval.lower, c.double("ciLower"), 1e-9) && close(interval.upper, c.double("ciUpper"), 1e-9))143            }144        }145    }146147    @Test func proportions() throws {148        for c in try Reference.cases("inference") where c.string("kind") == "proportionCI" {149            var s = InferenceState()150            s.procedure = .ciProportion; s.source = .summary; s.x1 = c.double("x"); s.trials1 = c.double("n"); s.level = c.double("level")151            for (method, key) in [(ProportionIntervalMethod.wilson, "wilson"), (.wald, "wald"), (.agrestiCoull, "agrestiCoull"), (.clopperPearson, "exact")] {152                s.proportionMethod = method153                let ci = try #require(try InferenceEngine.compute(s, dataset: Dataset()).interval)154                #expect(close(ci.lower, c.doubles(key)[0], 1e-8) && close(ci.upper, c.doubles(key)[1], 1e-8), "\(key)")155            }156            // The same counts from a 0/1 column.157            let x = c.int("x"), n = c.int("n")158            s.source = .data; s.proportionMethod = .wilson159            let ci = try #require(try InferenceEngine.compute(s, dataset: dataset(Array(repeating: 1, count: x) + Array(repeating: 0, count: n - x))).interval)160            #expect(close(ci.lower, c.doubles("wilson")[0], 1e-8))161        }162        #expect(InferenceState().proportionMethod == .wilson, "Wilson is the default")163        for c in try Reference.cases("inference") where c.string("kind") == "oneProportionZ" {164            var s = InferenceState()165            s.procedure = .onePropZ; s.source = .summary; s.x1 = c.double("x"); s.trials1 = c.double("n"); s.p0 = c.double("p0"); s.alternative = alternative(c.string("alternative"))166            let t = try #require(try InferenceEngine.compute(s, dataset: Dataset()).test)167            #expect(close(t.statistic, c.double("statistic"), 1e-9) && close(t.pValue, c.double("p"), 1e-8))168            s.continuity = true169            let cc = try #require(try InferenceEngine.compute(s, dataset: Dataset()).test)170            #expect(close(cc.statistic, c.double("statisticCC"), 1e-9) && close(cc.pValue, c.double("pCC"), 1e-8))171        }172        for c in try Reference.cases("inference") where c.string("kind") == "twoProportionZ" {173            var s = InferenceState()174            s.procedure = .twoPropZ; s.source = .summary; s.x1 = c.double("x1"); s.trials1 = c.double("n1"); s.x2 = c.double("x2"); s.trials2 = c.double("n2")175            let t = try #require(try InferenceEngine.compute(s, dataset: Dataset()).test)176            #expect(close(t.statistic, c.double("statistic"), 1e-9) && close(t.pValue, c.double("p"), 1e-8))177            s.alternative = .greater178            #expect(close(try #require(try InferenceEngine.compute(s, dataset: Dataset()).test).pValue, c.double("pGreater"), 1e-8))179            s.procedure = .ciTwoProportions180            let ci = try #require(try InferenceEngine.compute(s, dataset: Dataset()).interval)181            #expect(close(ci.lower, c.double("ciLower"), 1e-8) && close(ci.upper, c.double("ciUpper"), 1e-8))182        }183        var bad = InferenceState()184        bad.procedure = .onePropZ; bad.source = .data185        #expect(throws: InputError.self) { try InferenceEngine.compute(bad, dataset: self.dataset([0, 1, 2])) }186    }187188    @Test func chiSquareTests() throws {189        for c in try Reference.cases("inference") where c.string("kind") == "gof" {190            var s = InferenceState()191            s.procedure = .gof; s.source = .summary192            s.gofObserved = c.doubles("observed").map { String($0) }.joined(separator: " ")193            if c.has("proportions") {194                s.expectedKind = .proportions; s.gofExpected = c.doubles("proportions").map { String($0) }.joined(separator: ", ")195            }196            guard c.int("ddof") == 0 else { continue }197            let o = try InferenceEngine.compute(s, dataset: Dataset())198            let t = try #require(o.test)199            #expect(close(t.statistic, c.double("statistic"), 1e-9) && close(t.pValue, c.double("p"), 1e-8))200            #expect(close(t.criticalValue, c.double("critical"), 1e-9))201            #expect(zip(try #require(o.gof).expected, c.doubles("expected")).allSatisfy { close($0, $1, 1e-9) })202            // Counts given as expected counts, from columns.203            s.source = .data; s.expectedKind = .counts; s.columnA = 0; s.columnB = 1204            let fromColumns = try #require(try InferenceEngine.compute(s, dataset: dataset(c.doubles("observed"), c.doubles("expected"))).test)205            #expect(close(fromColumns.statistic, c.double("statistic"), 1e-9))206        }207        for c in try Reference.cases("inference") where c.string("kind") == "independence" {208            var s = InferenceState()209            s.procedure = .independence; s.source = .summary; s.yates = c.bool("yates")210            s.table = c.matrix("table").map { $0.map(Optional.some) }211            let o = try InferenceEngine.compute(s, dataset: Dataset())212            let r = try #require(o.independence)213            #expect(close(r.test.statistic, c.double("statistic"), 1e-9) && close(r.test.pValue, c.double("p"), 1e-8))214            #expect(close(r.cramersV, c.double("cramersV"), 1e-9))215            #expect(zip(r.expected.joined(), c.matrix("expected").joined()).allSatisfy { close($0, $1, 1e-9) })216            #expect(r.yatesApplied == c.bool("yatesApplied"))217        }218        // An empty cell is reported, not treated as zero.219        var s = InferenceState()220        s.procedure = .independence; s.source = .summary; s.table = [[12, nil], [7, 14]]221        #expect(throws: InputError.self) { try InferenceEngine.compute(s, dataset: Dataset()) }222        // Small expected counts carry the engine's warning.223        s.table = [[3, 2], [1, 4]]224        let small = try InferenceEngine.compute(s, dataset: Dataset())225        #expect(try #require(small.independence).expected.joined().contains { $0 < 5 })226        #expect(!small.verification.warnings.isEmpty)227        // Typed lists report what cannot be read.228        s.procedure = .gof; s.gofObserved = "18 22 x 15"229        #expect(throws: InputError.self) { try InferenceEngine.compute(s, dataset: Dataset()) }230    }231232    @Test func independenceFromTwoCategoryColumns() throws {233        // Rebuild the 2 × 2 table [[12, 5], [7, 14]] as 38 observations.234        var x: [Double] = [], y: [Double] = []235        for (r, row) in [[12, 5], [7, 14]].enumerated() { for (c, count) in row.enumerated() { for _ in 0..<count { x.append(Double(r)); y.append(Double(c + 10)) } } }236        #expect(try InferenceEngine.crossTab(x, y).table == [[12, 5], [7, 14]])237        var s = InferenceState()238        s.procedure = .independence; s.source = .data; s.columnA = 0; s.columnB = 1239        let r = try #require(try InferenceEngine.compute(s, dataset: dataset(x, y)).independence)240        #expect(close(r.test.statistic, 5.215686274509804, 1e-12))241        #expect(throws: InputError.self) { try InferenceEngine.crossTab([1, 1, 1], [1, 2, 3]) }242    }243244    @Test func anovaFTestCorrelationVariance() throws {245        for c in try Reference.cases("inference") where c.string("kind") == "anova" {246            let groups = c.matrix("groups")247            var s = InferenceState()248            s.procedure = .anova; s.source = .data; s.groupColumns = Array(groups.indices)249            let ds = Dataset(title: "g", columns: groups.enumerated().map { DataColumn(name: "G\($0.offset + 1)", values: $0.element.map(Optional.some)) })250            let a = try #require(try InferenceEngine.compute(s, dataset: ds).anova)251            #expect(close(a.test.statistic, c.double("statistic"), 1e-9) && close(a.test.pValue, c.double("p"), 1e-8))252            #expect(close(a.sumOfSquaresBetween, c.double("ssb"), 1e-9) && close(a.sumOfSquaresWithin, c.double("ssw"), 1e-9))253            #expect(close(a.meanSquareBetween, c.double("msb"), 1e-9) && close(a.meanSquareWithin, c.double("msw"), 1e-9))254            #expect(a.degreesOfFreedomBetween == c.double("df1") && a.degreesOfFreedomWithin == c.double("df2"))255            #expect(close(a.etaSquared, c.double("etaSquared"), 1e-9))256            s.source = .summary257            s.groups = c.matrix("summaries").map { GroupSummaryInput(n: $0[0], mean: $0[1], sd: $0[2]) }258            let fromSummary = try #require(try InferenceEngine.compute(s, dataset: Dataset()).anova)259            #expect(close(fromSummary.test.statistic, c.double("statistic"), 1e-8))260            s.source = .data; s.groupColumns = [0]261            #expect(throws: InputError.self) { try InferenceEngine.compute(s, dataset: ds) }262        }263        for c in try Reference.cases("inference") where c.string("kind") == "fTest" {264            var s = InferenceState()265            s.procedure = .fTest; s.source = .data; s.columnA = 0; s.columnB = 1266            let t = try #require(try InferenceEngine.compute(s, dataset: dataset(c.doubles("a"), c.doubles("b"))).test)267            #expect(close(t.statistic, c.double("statistic"), 1e-9) && close(t.pValue, c.double("p"), 1e-8))268            #expect(close(try #require(t.lowerCriticalValue), c.double("criticalLower"), 1e-9) && close(t.criticalValue, c.double("criticalUpper"), 1e-9))269            let a = summaryOf(c.doubles("a")), b = summaryOf(c.doubles("b"))270            s.source = .summary; s.n1 = a.n; s.sd1 = a.sd; s.n2 = b.n; s.sd2 = b.sd; s.alternative = .greater271            #expect(close(try #require(try InferenceEngine.compute(s, dataset: Dataset()).test).pValue, c.double("pGreater"), 1e-8))272        }273        for c in try Reference.cases("inference") where c.string("kind") == "correlation" {274            var s = InferenceState()275            s.procedure = .correlation; s.source = .data; s.columnA = 0; s.columnB = 1276            let t = try #require(try InferenceEngine.compute(s, dataset: dataset(c.doubles("x"), c.doubles("y"))).test)277            #expect(close(t.statistic, c.double("statistic"), 1e-9) && close(t.pValue, c.double("p"), 1e-8))278            s.source = .summary; s.r = c.double("r"); s.nPairs = Double(c.doubles("x").count); s.alternative = .less279            #expect(close(try #require(try InferenceEngine.compute(s, dataset: Dataset()).test).pValue, c.double("pLess"), 1e-8))280        }281        for c in try Reference.cases("inference") where c.string("kind") == "varianceCI" {282            var s = InferenceState()283            s.procedure = .ciVariance; s.source = .data; s.level = c.double("level")284            let o = try InferenceEngine.compute(s, dataset: dataset(c.doubles("data")))285            let ci = try #require(o.interval)286            #expect(close(ci.lower, c.double("lower"), 1e-8) && close(ci.upper, c.double("upper"), 1e-8))287            let sd = try #require(o.secondaryInterval)288            #expect(close(sd.lower, c.double("lower").squareRoot(), 1e-8))289            let sum = summaryOf(c.doubles("data"))290            s.source = .summary; s.n1 = sum.n; s.sd1 = sum.sd291            #expect(close(try #require(try InferenceEngine.compute(s, dataset: Dataset()).interval).upper, c.double("upper"), 1e-8))292        }293    }294295    @Test func everyProcedureComputesWithItsDefaults() throws {296        let ds = SampleDatasets.bloodPressure297        for p in InferenceProcedure.allCases {298            var s = InferenceState()299            s.procedure = p; s.source = .summary300            let o = try InferenceEngine.compute(s, dataset: ds)301            #expect(o.interval != nil || o.test != nil, "\(p)")302            #expect(o.procedure.isInterval == (o.test == nil))303            if let t = o.test {304                let plot = try #require(o.reference, "\(p)")305                #expect(plot.points.count > 100 && plot.points.contains { $0.rejected })306                // The rejection region has probability α under the reference distribution.307                let d = try #require(InferenceEngine.referenceDistribution(t))308                let mass = InferenceEngine.rejectionRegion(t).map { d.cdf($0.upperBound) - d.cdf($0.lowerBound) }.reduce(0, +)309                #expect(abs(mass - t.alpha) < 1e-9, "\(p): region mass \(mass)")310                // The statistic is in the region exactly when H₀ is rejected.311                #expect(InferenceEngine.rejectionRegion(t).contains { $0.contains(t.statistic) } == t.rejectNull, "\(p)")312            }313        }314        #expect(Set(InferenceProcedure.intervals + InferenceProcedure.tests) == Set(InferenceProcedure.allCases))315    }316317    @Test func inputErrorsAreExplained() {318        let m = InferenceModel()319        m.state.procedure = .tTest; m.state.source = .summary; m.state.n1 = 12.5320        #expect(m.calculate(dataset: Dataset(), dataRevision: 0) == nil)321        #expect(m.error?.contains("whole number") == true)322        #expect(!m.state.calculated)323        m.state.n1 = 12; m.state.sd1 = nil324        m.calculate(dataset: Dataset(), dataRevision: 0)325        #expect(m.error?.contains("standard deviation") == true)326        m.state.sd1 = 3; m.state.alpha = 1.5327        m.calculate(dataset: Dataset(), dataRevision: 0)328        #expect(m.error == StatisticsError.probabilityOutOfRange(1.5).description, "the engine's own message")329        m.state.alpha = 0.05; m.state.source = .data; m.state.columnA = 0330        m.calculate(dataset: Dataset(), dataRevision: 0)331        #expect(m.error?.contains("no values") == true)332    }333334    @Test func staleResultsAreFlagged() {335        let data = DataModel()336        data.loadDemo()337        let m = InferenceModel()338        m.loadDemo(dataset: data.dataset, dataRevision: data.revision)339        #expect(!m.isStale(dataRevision: data.revision))340        m.state.alpha = 0.01341        #expect(m.isStale(dataRevision: data.revision))342        m.calculate(dataset: data.dataset, dataRevision: data.revision)343        #expect(!m.isStale(dataRevision: data.revision))344        data.setValue(99, row: 0, column: 1)345        #expect(m.isStale(dataRevision: data.revision), "the dataset changed")346    }347348    @Test func calculateLogsToTheSharedHistory() throws {349        let app = makeApp()350        let store = StatisticsStore()351        store.data.loadDemo()352        store.inference.state.procedure = .tTest353        store.inference.state.source = .data; store.inference.state.columnA = 1; store.inference.state.mu0 = 65354        store.calculateInference(app)355        let entry = try #require(app.history.first)356        #expect(entry.mode == .statistics)357        #expect(entry.input.contains("t test") && entry.display.contains("H₀"))358        #expect(!entry.detail.isEmpty, "the explanation is stored")359        store.inference.state.procedure = .ciProportion; store.inference.state.source = .summary360        store.calculateInference(app)361        #expect(app.history.count == 2 && app.history.first?.display.contains("CI") == true)362        store.inference.state.trials1 = nil363        store.calculateInference(app)364        #expect(app.history.count == 2, "a failed calculation is not logged")365    }366}367
Files main
Menu

Theme