SPB Git

Repository Public MIT

DataTests.swift 13.7 KB 282 lines swift

1// ORIX Calc2// File: Packages/ORIXFeatureStatistics/Tests/ORIXFeatureStatisticsTests/DataTests.swift3// Description: Tests of the Data section's logic — paste parser, dataset rules, named datasets, variable import, descriptive results against independent and reference values.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Testing9import ORIXExpression10import ORIXStatistics11import ORIXUI12@testable import ORIXFeatureStatistics1314@Suite struct PasteParserTests {15    @Test func separators() {16        #expect(PasteParser.parse("1, 2, 3.5").columns == [[1, 2, 3.5]])17        #expect(PasteParser.parse("1 2\t3\n4\r\n5;6").columns == [[1, 2, 3, 4, 5, 6]])18        #expect(PasteParser.parse("1,2,3").columns == [[1, 2, 3]])19        #expect(PasteParser.parse(" 7 ").columns == [[7]])20        #expect(PasteParser.parse("1e3 2.5E-1 -4 +5 .5").columns == [[1000, 0.25, -4, 5, 0.5]])21        #expect(PasteParser.parse("−2 3").columns == [[-2, 3]], "typographic minus")22    }2324    @Test func emptyText() {25        let r = PasteParser.parse("  \n \t ")26        #expect(r.columns.isEmpty)27        #expect(r.valueCount == 0)28        #expect(r.unread.isEmpty)29    }3031    @Test func tables() {32        let t = PasteParser.parse("1\t10\n2\t20\n3\t30")33        #expect(t.columns == [[1, 2, 3], [10, 20, 30]])34        #expect(t.isTable)35        let csv = PasteParser.parse("x,y\n1,2\n3,4")36        #expect(csv.names == ["x", "y"])37        #expect(csv.columns == [[1, 3], [2, 4]])38        #expect(csv.unread.isEmpty, "a header line is not an unreadable token")39        // Ragged lines are a plain list, not a table.40        #expect(PasteParser.parse("1 2\n3\n4 5").columns == [[1, 2, 3, 4, 5]])41    }4243    @Test func decimalCommaWhenUnambiguous() {44        let semi = PasteParser.parse("1,5; 2,5; 3,5")45        #expect(semi.columns == [[1.5, 2.5, 3.5]])46        #expect(semi.commaAsDecimal && !semi.commaDecidedByLocale)47        let spaces = PasteParser.parse("1,5 2,5\t3,25")48        #expect(spaces.columns == [[1.5, 2.5, 3.25]])49        let table = PasteParser.parse("1,2 3,4\n5,6 7,8")50        #expect(table.columns == [[1.2, 5.6], [3.4, 7.8]])51        // A decimal point anywhere means commas separate values.52        #expect(PasteParser.parse("1.5,2.5").columns == [[1.5, 2.5]])53        #expect(!PasteParser.parse("1.5,2.5").commaAsDecimal)54        // A comma followed by a space separates values.55        #expect(PasteParser.parse("3, 14").columns == [[3, 14]])56    }5758    @Test func ambiguousCommaFollowsTheRegion() {59        let english = PasteParser.parse("3,14\n2,71", localeUsesDecimalComma: false)60        #expect(english.columns == [[3, 2], [14, 71]])61        #expect(english.commaDecidedByLocale && !english.commaAsDecimal)62        let french = PasteParser.parse("3,14\n2,71", localeUsesDecimalComma: true)63        #expect(french.columns == [[3.14, 2.71]])64        #expect(french.commaDecidedByLocale && french.commaAsDecimal)65    }6667    @Test func badTokensAreReported() {68        let r = PasteParser.parse("1, abc, 3, 4x, nan, inf, 5")69        #expect(r.columns == [[1, 3, 5]])70        #expect(r.unread == ["abc", "4x", "nan", "inf"])71        let t = PasteParser.parse("1\t2\n3\tn/a\n5\t6")72        #expect(t.columns == [[1, 3, 5], [2, nil, 6]], "a bad cell of a table stays empty")73        #expect(t.unread == ["n/a"])74    }75}7677@MainActor @Suite struct DataModelTests {78    @Test func pasteReplacesTheSelectedColumnOrTheTable() {79        let m = DataModel()80        m.load(SampleDatasets.studyHours)81        m.state.selected = 182        let report = m.paste("10 20 oops 30", localeUsesDecimalComma: false)83        #expect(report.values == 3 && report.unread == ["oops"])84        #expect(m.dataset.columns[1].values.prefix(4) == [10, 20, 30, nil])85        #expect(m.dataset.columns[0].values.count == 12, "the other column is untouched")86        #expect(m.pasteReport == report)8788        let table = m.paste("a\tb\tc\n1\t2\t3\n4\t5\t6", localeUsesDecimalComma: false)89        #expect(table.columns == 3 && table.values == 6)90        #expect(m.dataset.columnNames == ["a", "b", "c"])91        #expect(m.dataset.rowCount == 2)9293        let nothing = m.paste("hello world", localeUsesDecimalComma: false)94        #expect(nothing.isEmpty && nothing.unread == ["hello", "world"])95        #expect(m.dataset.rowCount == 2, "a paste without numbers changes nothing")96    }9798    @Test func editingRows() {99        let m = DataModel()100        m.setValue(3, row: 0, column: 0)101        m.setValue(5, row: 2, column: 0)102        #expect(m.dataset.rowCount == 3)103        #expect(m.analysis.summary.value?.count == 2, "the empty cell is ignored")104        m.deleteRow(1)105        #expect(m.dataset.columns[0].values == [3, 5])106        m.addColumn()107        #expect(m.dataset.columnNames == ["X", "Y"])108        m.setValue(1, row: 0, column: 1)109        m.state.paired = 1110        #expect(m.analysis.bivariate?.value?.count == 1)111        #expect(m.analysis.bivariate?.value?.skipped == 1)112        m.removeColumn(1)113        #expect(m.dataset.columns.count == 1 && m.state.paired == nil)114    }115116    @Test func thousandsOfRows() throws {117        let m = DataModel()118        let values = (1...20_000).map(Double.init)119        m.load(Dataset(title: "big", names: ["X"], columns: [values]))120        let s = try #require(m.analysis.summary.value)121        #expect(s.count == 20_000)122        #expect(s.mean == 10_000.5)123        m.setValue(nil, row: 19_999, column: 0)124        #expect(m.analysis.summary.value?.count == 19_999)125    }126127    @Test func frequencies() throws {128        var ds = SampleDatasets.dieRolls129        #expect(try ds.values(0).count == 120)130        #expect(close(try Descriptive.mean(ds.values(0)), (18 + 44 + 90 + 60 + 125 + 60) / 120.0))131        ds.setFrequency(2.5, row: 1)132        #expect(throws: DatasetError.badFrequency(row: 2, value: 2.5)) { try ds.values(0) }133        ds.setFrequency(nil, row: 1)134        #expect(try ds.values(0).count == 120 - 22 + 1, "an empty frequency counts as 1")135        ds.setFrequency(0, row: 0)136        #expect(try !ds.values(0).contains(1))137        let m = DataModel()138        m.load(SampleDatasets.dieRolls)139        m.setFrequency(-1, row: 0)140        #expect(m.analysis.summary.failure?.contains("Row 1") == true, "the error is shown plainly")141    }142143    @Test func demoMatchesIndependentCalculations() throws {144        let m = DataModel()145        m.loadDemo()146        let y: [Double] = [52, 58, 57, 64, 68, 66, 72, 79, 77, 83, 86, 93]147        let x: [Double] = [2, 3.5, 4, 5, 5.5, 6, 7, 8, 8.5, 9, 10, 11.5]148        let n = Double(y.count)149        let mean = y.reduce(0, +) / n, mx = x.reduce(0, +) / n150        let ss = y.map { ($0 - mean) * ($0 - mean) }.reduce(0, +)151        let sxy = zip(x, y).map { ($0 - mx) * ($1 - mean) }.reduce(0, +), sxx = x.map { ($0 - mx) * ($0 - mx) }.reduce(0, +)152        let s = try #require(m.analysis.summary.value)153        #expect(m.analysis.columnName == "Score")154        #expect(s.count == 12)155        #expect(close(s.sum, 855))156        #expect(close(s.mean, 71.25))157        #expect(close(s.median, 70))158        #expect(s.minimum == 52 && s.maximum == 93 && s.range == 41)159        #expect(close(try #require(s.sampleVariance), ss / (n - 1)))160        #expect(close(s.populationVariance, ss / n))161        #expect(close(try #require(s.sampleStandardDeviation), (ss / (n - 1)).squareRoot()))162        #expect(close(try #require(s.standardError), (ss / (n - 1)).squareRoot() / n.squareRoot()))163        #expect(s.modes.isEmpty)164        // R-7 quartiles of 12 sorted values: positions 2.75 and 8.25 (0-based).165        let sorted = y.sorted()166        #expect(close(s.fiveNumber.firstQuartile, sorted[2] + 0.75 * (sorted[3] - sorted[2])))167        #expect(close(s.fiveNumber.thirdQuartile, sorted[8] + 0.25 * (sorted[9] - sorted[8])))168        #expect(close(try #require(m.analysis.geometricMean.value), exp(y.map { log($0) }.reduce(0, +) / n)))169        #expect(close(try #require(m.analysis.harmonicMean.value), n / y.map { 1 / $0 }.reduce(0, +)))170        let b = try #require(m.analysis.bivariate?.value)171        #expect(close(try #require(b.sampleCovariance.value), sxy / (n - 1)))172        #expect(close(try #require(b.pearson.value), sxy / (sxx * ss).squareRoot()))173        // Three adjacent inversions among 12 ranks: ρ = 1 − 6·Σd²/(n(n² − 1)) with Σd² = 6.174        #expect(close(try #require(b.spearman.value), 1 - 36.0 / 1716, 1e-12))175        #expect(m.analysis.box?.outliers.isEmpty == true)176        #expect(m.analysis.histogram.value?.total == 12)177        // Percentile: default 90th with R-7.178        #expect(close(try #require(m.analysis.percentile?.value), sorted[9] + 0.9 * (sorted[10] - sorted[9])))179    }180181    @Test func domainErrorsAreShownPlainly() {182        let m = DataModel()183        m.load(Dataset(title: "t", names: ["X"], columns: [[-1, 2, 3]]))184        #expect(m.analysis.geometricMean.failure?.isEmpty == false)185        #expect(m.analysis.harmonicMean.failure?.isEmpty == false)186        #expect(m.analysis.summary.value != nil)187        m.state.percentile = 140188        #expect(m.analysis.percentile?.failure?.isEmpty == false)189        m.clear()190        #expect(m.analysis.summary.failure == StatisticsError.emptyData.description)191    }192193    @Test func referenceDescriptiveValues() throws {194        let cases = try Reference.cases("descriptive")195        #expect(cases.count >= 8)196        for c in cases {197            let m = DataModel()198            m.load(Dataset(title: c.string("name"), names: ["X"], columns: [c.doubles("data")]))199            let s = try #require(m.analysis.summary.value)200            #expect(s.count == c.int("n"))201            #expect(close(s.mean, c.double("mean"), 1e-12))202            #expect(close(s.median, c.double("median"), 1e-12))203            #expect(s.modes == c.doubles("modes"))204            #expect(close(s.populationVariance, c.double("populationVariance"), 1e-6))205            #expect(close(try #require(s.sampleStandardDeviation), c.double("sampleSD"), 1e-6))206            if c.has("sampleSkewness") {207                #expect(close(try #require(s.sampleSkewness), c.double("sampleSkewness"), 1e-9))208                #expect(close(try #require(s.sampleExcessKurtosis), c.double("sampleExcessKurtosis"), 1e-9))209                #expect(close(try #require(s.populationSkewness), c.double("populationSkewness"), 1e-9))210            }211            #expect(close(s.fiveNumber.interquartileRange, c.double("iqrLinear"), 1e-9))212            if c.has("geometricMean") { #expect(close(try #require(m.analysis.geometricMean.value), c.double("geometricMean"), 1e-10)) }213            m.state.quartiles = .tiExclusive214            let ti = try #require(m.analysis.summary.value).fiveNumber215            #expect(close(ti.firstQuartile, c.doubles("exclusiveHinges")[0], 1e-12) && close(ti.thirdQuartile, c.doubles("exclusiveHinges")[1], 1e-12))216            m.state.quartiles = .tukeyHinges217            let tukey = try #require(m.analysis.summary.value).fiveNumber218            #expect(close(tukey.firstQuartile, c.doubles("tukeyHinges")[0], 1e-12) && close(tukey.thirdQuartile, c.doubles("tukeyHinges")[1], 1e-12))219            m.state.binning = .sturges220            let h = try #require(m.analysis.histogram.value)221            let ref = (c["hist_sturges"] as? [String: Any]) ?? [:]222            #expect(h.counts == ref.doubles("counts").map { Int($0) })223            m.state.binning = .fixedCount; m.state.binCount = 7224            #expect(m.analysis.histogram.value?.counts == ((c["hist_fixed7"] as? [String: Any]) ?? [:]).doubles("counts").map { Int($0) })225        }226    }227228    @Test func namedDatasetRoundTrip() throws {229        let app = makeApp()230        let m = DataModel()231        var ds = SampleDatasets.housePrices232        ds.setValue(nil, row: 3, column: 2)233        ds.setFrequenciesEnabled(true)234        ds.setFrequency(2, row: 0)235        m.load(ds)236        let id = app.saveModel(kind: Stat.datasetKind, title: "Houses", payload: try #require(m.payload()))237        #expect(app.savedModels(kind: Stat.datasetKind).map(\.title) == ["Houses"])238239        let other = DataModel()240        let saved = try #require(app.savedModels(kind: Stat.datasetKind).first)241        #expect(other.open(saved.payload, title: saved.title))242        var expected = ds243        expected.title = "Houses"244        #expect(other.dataset == expected)245        #expect(other.dataset.columns[2].values[3] == nil)246        #expect(other.dataset.frequencies?[0] == 2)247        #expect(!other.open(Data("not json".utf8), title: "x"), "a damaged payload is refused")248        #expect(other.dataset == expected)249250        app.deleteModel(id)251        #expect(app.savedModels(kind: Stat.datasetKind).isEmpty)252    }253254    @Test func importsCalculatorVectors() async throws {255        let app = makeApp()256        _ = await app.evaluateCanonical("v = [4, 8, 15, 16, 23, 42]", mode: .scientific)257        _ = await app.evaluateCanonical("k = 3", mode: .scientific)258        _ = await app.evaluateCanonical("M = [[1, 2], [3, 4], [5, 6]]", mode: .scientific)259        let variables = ImportableVariable.all(in: app.engine.context.variables)260        let v = try #require(variables.first { $0.name == "v" })261        #expect(v.columns == [[4, 8, 15, 16, 23, 42]])262        #expect(!variables.contains { $0.name == "k" }, "scalars are not datasets")263        let m = DataModel()264        m.importVariable(v)265        #expect(m.analysis.summary.value?.sum == 108)266        if let matrix = variables.first(where: { $0.name == "M" }) {267            #expect(matrix.columns == [[1, 3, 5], [2, 4, 6]])268            m.importVariable(matrix)269            #expect(m.dataset.columns.count == 2 && m.dataset.rowCount == 3)270        }271    }272273    @Test func sampleDatasetsAreUsable() throws {274        for ds in SampleDatasets.all {275            let m = DataModel()276            m.load(ds)277            #expect(m.analysis.summary.value != nil, "\(ds.title)")278            #expect(ds.columns.allSatisfy { $0.values.count == ds.rowCount })279        }280    }281}282
Files main
Menu

Theme