Repository Public
1import Foundation2import Testing3import SheetCore4import SheetUI5import Statistics6import AnalysisKit7@testable import StatsModels89@MainActor @Suite struct LinearRegressionTests {10 let d = Synth()1112 @Test func olsMatchesTheEngine() throws {13 let s = session(Synth.header, d.rows)14 let r = try run(LinearRegressionTool.descriptor, s, range: "A1:F61") { $0.set("y", column: 0); $0.set("x", columns: [1, 2]); $0.set("cat", columns: []) }15 let ref = try LinearRegression.fit(y: d.y, columns: [d.x1, d.x2], names: ["x1", "x2"])16 let coef = try #require(r.table(titled: "Coefficients"))17 #expect(coef.rows.map { $0[0].textValue } == ["Term", "Intercept", "x1", "x2"])18 for i in 0..<3 {19 #expect(close(coef.number(i + 1, 1), ref.coefficients[i]))20 #expect(close(coef.number(i + 1, 2), ref.standardErrors[i]))21 #expect(close(coef.number(i + 1, 4), ref.pValues[i]))22 #expect(close(coef.number(i + 1, 5), ref.ciLower[i]) && close(coef.number(i + 1, 6), ref.ciUpper[i]))23 }24 #expect(coef.rows[0][3].textValue == "t")25 let fit = try #require(r.table(titled: "Fit statistics"))26 #expect(close(fit.value("R²"), ref.rSquared) && close(fit.value("AIC"), ref.aic) && close(fit.value("Condition number"), ref.conditionNumber))27 #expect(close(r.metric("Observations") ?? 0, 60))28 let diag = try #require(r.table(titled: "Diagnostics"))29 #expect(close(diag.value("Breusch–Pagan (studentized)"), try RegressionDiagnostics.breuschPagan(ref).statistic))30 #expect(close(diag.value("Durbin–Watson"), RegressionDiagnostics.durbinWatson(ref.residuals)))31 #expect(close(diag.value("Breusch–Godfrey (2 lags)", 3), try RegressionDiagnostics.breuschGodfrey(ref, lags: 2).pValue ?? .nan))32 #expect(close(diag.value("Ramsey RESET (powers 2–3)"), try RegressionDiagnostics.reset(ref).statistic))33 let bp = try RegressionDiagnostics.breuschPagan(ref).pValue ?? 134 #expect(diag.text("Breusch–Pagan (studentized)", 4) == (bp < 0.05 ? "Reject H₀" : "Fail to reject H₀"))35 let vif = try #require(r.table(titled: "Variance inflation factors"))36 #expect(close(vif.value("x1"), ref.vif[1] ?? .nan))37 #expect(r.charts.count == 4)38 }3940 @Test func categoricalDummiesAndListwiseDeletion() throws {41 var rows = d.rows42 rows[4][1] = nil // missing regressor43 rows[9][0] = "n/a" // text in y44 rows[20][3] = nil // missing category45 let s = session(Synth.header, rows)46 let r = try run(LinearRegressionTool.descriptor, s, range: "A1:F61") { $0.set("y", column: 0); $0.set("x", columns: [1, 2]); $0.set("cat", columns: [3]) }47 let keep = d.y.indices.filter { ![4, 9, 20].contains($0) }48 var b = DesignBuilder()49 b.addNumeric("x1", keep.map { d.x1[$0] }); b.addNumeric("x2", keep.map { d.x2[$0] })50 try b.addCategorical("g", keep.map { d.g[$0] })51 let ref = try LinearRegression.fit(y: keep.map { d.y[$0] }, x: try b.matrix(), names: b.names)52 let coef = try #require(r.table(titled: "Coefficients"))53 #expect(coef.rows.dropFirst().map { $0[0].textValue ?? "" } == ["Intercept", "x1", "x2", "g[b]", "g[c]"])54 for i in ref.coefficients.indices { #expect(close(coef.number(i + 1, 1), ref.coefficients[i], 1e-9)) }55 #expect(r.metric("Observations") == 57)56 let notes = r.blocks.compactMap { b -> [String]? in if case .text(let t) = b, t.kind == .note { t.lines } else { nil } }.flatMap { $0 }57 #expect(notes.contains { $0.contains("57 of 60") })58 #expect(notes.contains { $0.contains("reference level: a") })59 }6061 @Test func weightedAndRobustCovariances() throws {62 let s = session(Synth.header, d.rows)63 let wls = try run(LinearRegressionTool.descriptor, s, range: "A1:F61") {64 $0.set("y", column: 0); $0.set("x", columns: [1, 2]); $0.set("cat", columns: []); $0.set("weights", column: 4); $0.set("cov", choice: "hc3")65 }66 let ref = try LinearRegression.fit(y: d.y, columns: [d.x1, d.x2], weights: d.w, covariance: .hc3)67 let coef = try #require(wls.table(titled: "Coefficients"))68 #expect(close(coef.number(2, 1), ref.coefficients[1]) && close(coef.number(2, 2), ref.standardErrors[1]))69 #expect(coef.rows[0][3].textValue == "z")70 #expect(wls.title.contains("WLS"))7172 let hac = try run(LinearRegressionTool.descriptor, s, range: "A1:F61") {73 $0.set("y", column: 0); $0.set("x", columns: [1, 2]); $0.set("cat", columns: []); $0.set("cov", choice: "hac"); $0.set("hacLags", int: 3)74 }75 let refHAC = try LinearRegression.fit(y: d.y, columns: [d.x1, d.x2], covariance: .hac(maxLags: 3, smallSample: true))76 #expect(close(try #require(hac.table(titled: "Coefficients")).number(3, 2), refHAC.standardErrors[2]))7778 let cl = try run(LinearRegressionTool.descriptor, s, range: "A1:F61") {79 $0.set("y", column: 0); $0.set("x", columns: [1, 2]); $0.set("cat", columns: []); $0.set("cov", choice: "cluster"); $0.set("cluster", column: 3)80 }81 let codes = d.g.map { ["a": 0, "b": 1, "c": 2][$0]! }82 let refCl = try LinearRegression.fit(y: d.y, columns: [d.x1, d.x2], covariance: .cluster(groups: codes, smallSample: true))83 #expect(close(try #require(cl.table(titled: "Coefficients")).number(2, 2), refCl.standardErrors[1]))84 }8586 @Test func fittedValuesAndResidualsAreWrittenOnTheRightRowsAndUndone() throws {87 var rows = d.rows88 rows[2][1] = nil89 let s = session(Synth.header, rows)90 let r = try run(LinearRegressionTool.descriptor, s, range: "A1:C61") { $0.set("y", column: 0); $0.set("x", columns: [1, 2]); $0.set("cat", columns: []) }91 let keep = d.y.indices.filter { $0 != 2 }92 let ref = try LinearRegression.fit(y: keep.map { d.y[$0] }, columns: [keep.map { d.x1[$0] }, keep.map { d.x2[$0] }])93 let before = snapshot(s.workbook)94 let action = try #require(r.actions.first { $0.id == "write-fitted" })95 let status = try action.perform(s)96 // columns D–F hold data, so the block goes to G:H97 let sh = s.workbook.sheets[0]98 #expect(status?.contains("G1:H61") == true)99 #expect(sh.value(0, 6) == .text("Fitted y") && sh.value(0, 7) == .text("Residual y"))100 #expect(close(sh.value(1, 6).numberValue ?? .nan, ref.fitted[0]) && close(sh.value(1, 7).numberValue ?? .nan, ref.residuals[0]))101 #expect(sh.value(3, 6).isBlank && sh.value(3, 7).isBlank) // the incomplete row stays empty102 #expect(close(sh.value(60, 7).numberValue ?? .nan, ref.residuals[58]))103 s.undo()104 #expect(snapshot(s.workbook) == before && !s.engine.canUndo)105 }106107 @Test func errorPaths() throws {108 // rank deficient: x2 is a multiple of x1109 var rows = [[Any?]]()110 for i in 0..<12 {111 let x = Double(i), y: Double = x * 1.5 + Double(i % 3), x2: Double = x * 2112 rows.append([y, x, x2])113 }114 let s = session(["y", "x1", "x2"], rows)115 #expect(throws: (any Error).self) { try run(LinearRegressionTool.descriptor, s, range: "A1:C13") { $0.set("y", column: 0); $0.set("x", columns: [1, 2]) } }116 // too few observations117 let tiny = session(["y", "x"], [[1.0, 2.0], [2.0, 1.0]])118 #expect(throws: ToolError.self) { try run(LinearRegressionTool.descriptor, tiny, range: "A1:B3") { $0.set("y", column: 0); $0.set("x", columns: [1]) } }119 // no regressor120 #expect(throws: ToolError.self) { try run(LinearRegressionTool.descriptor, s, range: "A1:C13") { $0.set("y", column: 0); $0.set("x", columns: [0]) } }121 }122}123124@MainActor @Suite struct PanelAndLogisticTests {125 @Test func fixedEffectsMatchTheEngine() throws {126 var rng = Xoshiro256StarStar(seed: 9)127 var rows = [[Any?]](), y = [Double](), x = [Double](), firm = [String](), year = [Double]()128 for f in 0..<6 {129 let effect = 5 * rng.nextNormal()130 for t in 0..<5 {131 let xv = 10 * rng.nextUniform(), yv = effect + 1.5 * xv + 0.3 * Double(t) + rng.nextNormal()132 y.append(yv); x.append(xv); firm.append("F\(f)"); year.append(Double(2020 + t))133 rows.append(["F\(f)", 2020 + t, yv, xv])134 }135 }136 let s = session(["firm", "year", "y", "x"], rows)137 let r = try run(PanelRegressionTool.descriptor, s, range: "A1:D31") {138 $0.set("y", column: 2); $0.set("x", columns: [3]); $0.set("entity", column: 0); $0.set("time", column: 1)139 $0.set("timeEffects", bool: true); $0.set("cov", choice: "cluster")140 }141 let ref = try PanelRegression.fixedEffects(y: y, x: Matrix(columns: [x]), names: ["x"], entity: firm, time: year, timeEffects: true,142 covariance: .clusteredByEntity(correction: .linearmodels))143 let coef = try #require(r.table(titled: "Coefficients"))144 #expect(close(coef.number(1, 1), ref.coefficients[0], 1e-9) && close(coef.number(1, 2), ref.standardErrors[0], 1e-9))145 let fit = try #require(r.table(titled: "Fit statistics"))146 #expect(close(fit.value("Within R²"), ref.rSquared, 1e-9) && close(fit.value("Poolability F (all effects equal)"), ref.poolabilityF, 1e-9))147 #expect(fit.value("Entities") == 6 && fit.value("Periods") == 5)148 let effects = try #require(r.table(titled: "Estimated entity effects"))149 #expect(effects.rows.count == 7 && close(effects.value("F0"), ref.entityEffects[0], 1e-9))150 #expect(r.table(titled: "Estimated time effects")?.rows.count == 6)151152 // entity effects only, classical covariance, no time column153 let plain = try run(PanelRegressionTool.descriptor, s, range: "A1:D31") { $0.set("y", column: 2); $0.set("x", columns: [3]); $0.set("entity", column: 0); $0.set("time", column: nil) }154 let refPlain = try PanelRegression.fixedEffects(y: y, x: Matrix(columns: [x]), entity: firm)155 #expect(close(try #require(plain.table(titled: "Coefficients")).number(1, 1), refPlain.coefficients[0], 1e-9))156 #expect(plain.table(titled: "Estimated time effects") == nil)157 }158159 @Test func logisticMatchesTheEngineAndRejectsBadOutcomes() throws {160 let d = Synth(n: 120, seed: 5)161 let s = session(Synth.header, d.rows)162 let r = try run(LogisticRegressionTool.descriptor, s, range: "A1:F121") { $0.set("y", column: 5); $0.set("x", columns: [1, 2]); $0.set("cat", columns: []) }163 let ref = try LogisticRegression.fit(y: d.b, x: Matrix(columns: [d.x1, d.x2]), names: ["x1", "x2"])164 let coef = try #require(r.table(titled: "Coefficients (log-odds)"))165 for i in 0..<3 { #expect(close(coef.number(i + 1, 1), ref.coefficients[i], 1e-9) && close(coef.number(i + 1, 4), ref.pValues[i], 1e-9)) }166 let odds = try #require(r.table(titled: "Odds ratios"))167 #expect(close(odds.value("x1"), ref.oddsRatios[1], 1e-9) && close(odds.value("x1", 2), exp(ref.ciLower[1]), 1e-9))168 let ct = try #require(r.table(containing: "Classification table"))169 #expect(ct.number(1, 1) == Double(ref.classificationTable[0][0]) && ct.number(2, 2) == Double(ref.classificationTable[1][1]))170 #expect(close(ct.number(3, 3), ref.accuracy))171 #expect(close(r.metric("McFadden pseudo-R²") ?? .nan, ref.pseudoRSquared, 1e-9))172173 // a dependent variable that is not 0/1174 #expect(throws: ToolError.self) { try run(LogisticRegressionTool.descriptor, s, range: "A1:F121") { $0.set("y", column: 0); $0.set("x", columns: [1]) } }175 // complete separation → the engine's "did not converge"176 var sepRows = [[Any?]]()177 for i in 0..<20 { sepRows.append([i < 10 ? 0 : 1, Double(i)]) }178 let sep = session(["y", "x"], sepRows)179 do {180 _ = try run(LogisticRegressionTool.descriptor, sep, range: "A1:B21") { $0.set("y", column: 0); $0.set("x", columns: [1]) }181 Issue.record("separation should not converge")182 } catch { #expect(ToolError.describe(error).lowercased().contains("converge")) }183 }184}185