Repository Public MIT
1// ORIX Calc2// File: Packages/ORIXFeatureStatistics/Tests/ORIXFeatureStatisticsTests/DataTests.swift3// Description: Tests of the Data section's logic — paste parser, dataset rules, named datasets, variable import, descriptive results against independent and reference values.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Testing9import ORIXExpression10import ORIXStatistics11import ORIXUI12@testable import ORIXFeatureStatistics1314@Suite struct PasteParserTests {15 @Test func separators() {16 #expect(PasteParser.parse("1, 2, 3.5").columns == [[1, 2, 3.5]])17 #expect(PasteParser.parse("1 2\t3\n4\r\n5;6").columns == [[1, 2, 3, 4, 5, 6]])18 #expect(PasteParser.parse("1,2,3").columns == [[1, 2, 3]])19 #expect(PasteParser.parse(" 7 ").columns == [[7]])20 #expect(PasteParser.parse("1e3 2.5E-1 -4 +5 .5").columns == [[1000, 0.25, -4, 5, 0.5]])21 #expect(PasteParser.parse("−2 3").columns == [[-2, 3]], "typographic minus")22 }2324 @Test func emptyText() {25 let r = PasteParser.parse(" \n \t ")26 #expect(r.columns.isEmpty)27 #expect(r.valueCount == 0)28 #expect(r.unread.isEmpty)29 }3031 @Test func tables() {32 let t = PasteParser.parse("1\t10\n2\t20\n3\t30")33 #expect(t.columns == [[1, 2, 3], [10, 20, 30]])34 #expect(t.isTable)35 let csv = PasteParser.parse("x,y\n1,2\n3,4")36 #expect(csv.names == ["x", "y"])37 #expect(csv.columns == [[1, 3], [2, 4]])38 #expect(csv.unread.isEmpty, "a header line is not an unreadable token")39 // Ragged lines are a plain list, not a table.40 #expect(PasteParser.parse("1 2\n3\n4 5").columns == [[1, 2, 3, 4, 5]])41 }4243 @Test func decimalCommaWhenUnambiguous() {44 let semi = PasteParser.parse("1,5; 2,5; 3,5")45 #expect(semi.columns == [[1.5, 2.5, 3.5]])46 #expect(semi.commaAsDecimal && !semi.commaDecidedByLocale)47 let spaces = PasteParser.parse("1,5 2,5\t3,25")48 #expect(spaces.columns == [[1.5, 2.5, 3.25]])49 let table = PasteParser.parse("1,2 3,4\n5,6 7,8")50 #expect(table.columns == [[1.2, 5.6], [3.4, 7.8]])51 // A decimal point anywhere means commas separate values.52 #expect(PasteParser.parse("1.5,2.5").columns == [[1.5, 2.5]])53 #expect(!PasteParser.parse("1.5,2.5").commaAsDecimal)54 // A comma followed by a space separates values.55 #expect(PasteParser.parse("3, 14").columns == [[3, 14]])56 }5758 @Test func ambiguousCommaFollowsTheRegion() {59 let english = PasteParser.parse("3,14\n2,71", localeUsesDecimalComma: false)60 #expect(english.columns == [[3, 2], [14, 71]])61 #expect(english.commaDecidedByLocale && !english.commaAsDecimal)62 let french = PasteParser.parse("3,14\n2,71", localeUsesDecimalComma: true)63 #expect(french.columns == [[3.14, 2.71]])64 #expect(french.commaDecidedByLocale && french.commaAsDecimal)65 }6667 @Test func badTokensAreReported() {68 let r = PasteParser.parse("1, abc, 3, 4x, nan, inf, 5")69 #expect(r.columns == [[1, 3, 5]])70 #expect(r.unread == ["abc", "4x", "nan", "inf"])71 let t = PasteParser.parse("1\t2\n3\tn/a\n5\t6")72 #expect(t.columns == [[1, 3, 5], [2, nil, 6]], "a bad cell of a table stays empty")73 #expect(t.unread == ["n/a"])74 }75}7677@MainActor @Suite struct DataModelTests {78 @Test func pasteReplacesTheSelectedColumnOrTheTable() {79 let m = DataModel()80 m.load(SampleDatasets.studyHours)81 m.state.selected = 182 let report = m.paste("10 20 oops 30", localeUsesDecimalComma: false)83 #expect(report.values == 3 && report.unread == ["oops"])84 #expect(m.dataset.columns[1].values.prefix(4) == [10, 20, 30, nil])85 #expect(m.dataset.columns[0].values.count == 12, "the other column is untouched")86 #expect(m.pasteReport == report)8788 let table = m.paste("a\tb\tc\n1\t2\t3\n4\t5\t6", localeUsesDecimalComma: false)89 #expect(table.columns == 3 && table.values == 6)90 #expect(m.dataset.columnNames == ["a", "b", "c"])91 #expect(m.dataset.rowCount == 2)9293 let nothing = m.paste("hello world", localeUsesDecimalComma: false)94 #expect(nothing.isEmpty && nothing.unread == ["hello", "world"])95 #expect(m.dataset.rowCount == 2, "a paste without numbers changes nothing")96 }9798 @Test func editingRows() {99 let m = DataModel()100 m.setValue(3, row: 0, column: 0)101 m.setValue(5, row: 2, column: 0)102 #expect(m.dataset.rowCount == 3)103 #expect(m.analysis.summary.value?.count == 2, "the empty cell is ignored")104 m.deleteRow(1)105 #expect(m.dataset.columns[0].values == [3, 5])106 m.addColumn()107 #expect(m.dataset.columnNames == ["X", "Y"])108 m.setValue(1, row: 0, column: 1)109 m.state.paired = 1110 #expect(m.analysis.bivariate?.value?.count == 1)111 #expect(m.analysis.bivariate?.value?.skipped == 1)112 m.removeColumn(1)113 #expect(m.dataset.columns.count == 1 && m.state.paired == nil)114 }115116 @Test func thousandsOfRows() throws {117 let m = DataModel()118 let values = (1...20_000).map(Double.init)119 m.load(Dataset(title: "big", names: ["X"], columns: [values]))120 let s = try #require(m.analysis.summary.value)121 #expect(s.count == 20_000)122 #expect(s.mean == 10_000.5)123 m.setValue(nil, row: 19_999, column: 0)124 #expect(m.analysis.summary.value?.count == 19_999)125 }126127 @Test func frequencies() throws {128 var ds = SampleDatasets.dieRolls129 #expect(try ds.values(0).count == 120)130 #expect(close(try Descriptive.mean(ds.values(0)), (18 + 44 + 90 + 60 + 125 + 60) / 120.0))131 ds.setFrequency(2.5, row: 1)132 #expect(throws: DatasetError.badFrequency(row: 2, value: 2.5)) { try ds.values(0) }133 ds.setFrequency(nil, row: 1)134 #expect(try ds.values(0).count == 120 - 22 + 1, "an empty frequency counts as 1")135 ds.setFrequency(0, row: 0)136 #expect(try !ds.values(0).contains(1))137 let m = DataModel()138 m.load(SampleDatasets.dieRolls)139 m.setFrequency(-1, row: 0)140 #expect(m.analysis.summary.failure?.contains("Row 1") == true, "the error is shown plainly")141 }142143 @Test func demoMatchesIndependentCalculations() throws {144 let m = DataModel()145 m.loadDemo()146 let y: [Double] = [52, 58, 57, 64, 68, 66, 72, 79, 77, 83, 86, 93]147 let x: [Double] = [2, 3.5, 4, 5, 5.5, 6, 7, 8, 8.5, 9, 10, 11.5]148 let n = Double(y.count)149 let mean = y.reduce(0, +) / n, mx = x.reduce(0, +) / n150 let ss = y.map { ($0 - mean) * ($0 - mean) }.reduce(0, +)151 let sxy = zip(x, y).map { ($0 - mx) * ($1 - mean) }.reduce(0, +), sxx = x.map { ($0 - mx) * ($0 - mx) }.reduce(0, +)152 let s = try #require(m.analysis.summary.value)153 #expect(m.analysis.columnName == "Score")154 #expect(s.count == 12)155 #expect(close(s.sum, 855))156 #expect(close(s.mean, 71.25))157 #expect(close(s.median, 70))158 #expect(s.minimum == 52 && s.maximum == 93 && s.range == 41)159 #expect(close(try #require(s.sampleVariance), ss / (n - 1)))160 #expect(close(s.populationVariance, ss / n))161 #expect(close(try #require(s.sampleStandardDeviation), (ss / (n - 1)).squareRoot()))162 #expect(close(try #require(s.standardError), (ss / (n - 1)).squareRoot() / n.squareRoot()))163 #expect(s.modes.isEmpty)164 // R-7 quartiles of 12 sorted values: positions 2.75 and 8.25 (0-based).165 let sorted = y.sorted()166 #expect(close(s.fiveNumber.firstQuartile, sorted[2] + 0.75 * (sorted[3] - sorted[2])))167 #expect(close(s.fiveNumber.thirdQuartile, sorted[8] + 0.25 * (sorted[9] - sorted[8])))168 #expect(close(try #require(m.analysis.geometricMean.value), exp(y.map { log($0) }.reduce(0, +) / n)))169 #expect(close(try #require(m.analysis.harmonicMean.value), n / y.map { 1 / $0 }.reduce(0, +)))170 let b = try #require(m.analysis.bivariate?.value)171 #expect(close(try #require(b.sampleCovariance.value), sxy / (n - 1)))172 #expect(close(try #require(b.pearson.value), sxy / (sxx * ss).squareRoot()))173 // Three adjacent inversions among 12 ranks: ρ = 1 − 6·Σd²/(n(n² − 1)) with Σd² = 6.174 #expect(close(try #require(b.spearman.value), 1 - 36.0 / 1716, 1e-12))175 #expect(m.analysis.box?.outliers.isEmpty == true)176 #expect(m.analysis.histogram.value?.total == 12)177 // Percentile: default 90th with R-7.178 #expect(close(try #require(m.analysis.percentile?.value), sorted[9] + 0.9 * (sorted[10] - sorted[9])))179 }180181 @Test func domainErrorsAreShownPlainly() {182 let m = DataModel()183 m.load(Dataset(title: "t", names: ["X"], columns: [[-1, 2, 3]]))184 #expect(m.analysis.geometricMean.failure?.isEmpty == false)185 #expect(m.analysis.harmonicMean.failure?.isEmpty == false)186 #expect(m.analysis.summary.value != nil)187 m.state.percentile = 140188 #expect(m.analysis.percentile?.failure?.isEmpty == false)189 m.clear()190 #expect(m.analysis.summary.failure == StatisticsError.emptyData.description)191 }192193 @Test func referenceDescriptiveValues() throws {194 let cases = try Reference.cases("descriptive")195 #expect(cases.count >= 8)196 for c in cases {197 let m = DataModel()198 m.load(Dataset(title: c.string("name"), names: ["X"], columns: [c.doubles("data")]))199 let s = try #require(m.analysis.summary.value)200 #expect(s.count == c.int("n"))201 #expect(close(s.mean, c.double("mean"), 1e-12))202 #expect(close(s.median, c.double("median"), 1e-12))203 #expect(s.modes == c.doubles("modes"))204 #expect(close(s.populationVariance, c.double("populationVariance"), 1e-6))205 #expect(close(try #require(s.sampleStandardDeviation), c.double("sampleSD"), 1e-6))206 if c.has("sampleSkewness") {207 #expect(close(try #require(s.sampleSkewness), c.double("sampleSkewness"), 1e-9))208 #expect(close(try #require(s.sampleExcessKurtosis), c.double("sampleExcessKurtosis"), 1e-9))209 #expect(close(try #require(s.populationSkewness), c.double("populationSkewness"), 1e-9))210 }211 #expect(close(s.fiveNumber.interquartileRange, c.double("iqrLinear"), 1e-9))212 if c.has("geometricMean") { #expect(close(try #require(m.analysis.geometricMean.value), c.double("geometricMean"), 1e-10)) }213 m.state.quartiles = .tiExclusive214 let ti = try #require(m.analysis.summary.value).fiveNumber215 #expect(close(ti.firstQuartile, c.doubles("exclusiveHinges")[0], 1e-12) && close(ti.thirdQuartile, c.doubles("exclusiveHinges")[1], 1e-12))216 m.state.quartiles = .tukeyHinges217 let tukey = try #require(m.analysis.summary.value).fiveNumber218 #expect(close(tukey.firstQuartile, c.doubles("tukeyHinges")[0], 1e-12) && close(tukey.thirdQuartile, c.doubles("tukeyHinges")[1], 1e-12))219 m.state.binning = .sturges220 let h = try #require(m.analysis.histogram.value)221 let ref = (c["hist_sturges"] as? [String: Any]) ?? [:]222 #expect(h.counts == ref.doubles("counts").map { Int($0) })223 m.state.binning = .fixedCount; m.state.binCount = 7224 #expect(m.analysis.histogram.value?.counts == ((c["hist_fixed7"] as? [String: Any]) ?? [:]).doubles("counts").map { Int($0) })225 }226 }227228 @Test func namedDatasetRoundTrip() throws {229 let app = makeApp()230 let m = DataModel()231 var ds = SampleDatasets.housePrices232 ds.setValue(nil, row: 3, column: 2)233 ds.setFrequenciesEnabled(true)234 ds.setFrequency(2, row: 0)235 m.load(ds)236 let id = app.saveModel(kind: Stat.datasetKind, title: "Houses", payload: try #require(m.payload()))237 #expect(app.savedModels(kind: Stat.datasetKind).map(\.title) == ["Houses"])238239 let other = DataModel()240 let saved = try #require(app.savedModels(kind: Stat.datasetKind).first)241 #expect(other.open(saved.payload, title: saved.title))242 var expected = ds243 expected.title = "Houses"244 #expect(other.dataset == expected)245 #expect(other.dataset.columns[2].values[3] == nil)246 #expect(other.dataset.frequencies?[0] == 2)247 #expect(!other.open(Data("not json".utf8), title: "x"), "a damaged payload is refused")248 #expect(other.dataset == expected)249250 app.deleteModel(id)251 #expect(app.savedModels(kind: Stat.datasetKind).isEmpty)252 }253254 @Test func importsCalculatorVectors() async throws {255 let app = makeApp()256 _ = await app.evaluateCanonical("v = [4, 8, 15, 16, 23, 42]", mode: .scientific)257 _ = await app.evaluateCanonical("k = 3", mode: .scientific)258 _ = await app.evaluateCanonical("M = [[1, 2], [3, 4], [5, 6]]", mode: .scientific)259 let variables = ImportableVariable.all(in: app.engine.context.variables)260 let v = try #require(variables.first { $0.name == "v" })261 #expect(v.columns == [[4, 8, 15, 16, 23, 42]])262 #expect(!variables.contains { $0.name == "k" }, "scalars are not datasets")263 let m = DataModel()264 m.importVariable(v)265 #expect(m.analysis.summary.value?.sum == 108)266 if let matrix = variables.first(where: { $0.name == "M" }) {267 #expect(matrix.columns == [[1, 3, 5], [2, 4, 6]])268 m.importVariable(matrix)269 #expect(m.dataset.columns.count == 2 && m.dataset.rowCount == 3)270 }271 }272273 @Test func sampleDatasetsAreUsable() throws {274 for ds in SampleDatasets.all {275 let m = DataModel()276 m.load(ds)277 #expect(m.analysis.summary.value != nil, "\(ds.title)")278 #expect(ds.columns.allSatisfy { $0.values.count == ds.rowCount })279 }280 }281}282