Repository Public MIT
1// ORIX Calc2// File: Packages/ORIXStatistics/Tests/ORIXStatisticsTests/RegressionTests.swift3// Description: Simple, multiple and curve-fit regression against statsmodels / scipy / numpy and a 60-digit solve (Longley included).4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Testing9import ORIXNumerics10@testable import ORIXStatistics1112@Suite struct RegressionReferenceTests {13 static func cases(_ kind: String) throws -> [[String: Any]] {14 let all = try Reference.cases("regression").filter { $0.string("kind") == kind }15 #expect(!all.isEmpty)16 return all17 }1819 @Test func simpleRegressionMatchesLinregressAndStatsmodels() throws {20 for c in try Self.cases("simple") {21 let x = c.doubles("x"), y = c.doubles("y"), name = c.string("name")22 let r = try LinearRegression.simple(x: x, y: y, level: c.double("level"))23 // Coefficients and standard errors against the 60-digit solution, and against scipy.stats.linregress.24 #expect(close(r.slope.estimate, c.double("slope"), rtol: 1e-12) && close(r.intercept.estimate, c.double("intercept"), rtol: 1e-11), "\(name)")25 #expect(close(r.slope.standardError, c.double("slopeSE"), rtol: 1e-10) && close(r.intercept.standardError, c.double("interceptSE"), rtol: 1e-10), "\(name) SE")26 #expect(close(r.slope.estimate, c.double("linregressSlope"), rtol: 1e-11) && close(r.intercept.estimate, c.double("linregressIntercept"), rtol: 1e-10))27 #expect(close(r.slope.standardError, c.double("linregressSlopeSE"), rtol: 1e-9) && close(r.intercept.standardError, c.double("linregressInterceptSE"), rtol: 1e-9))28 #expect(close(r.slope.tStatistic, c.double("tSlope"), rtol: 1e-10) && close(r.intercept.tStatistic, c.double("tIntercept"), rtol: 1e-10))29 #expect(close(r.slope.pValue, c.double("pSlope"), rtol: 1e-8, atol: 1e-300) && close(r.intercept.pValue, c.double("pIntercept"), rtol: 1e-8, atol: 1e-300), "\(name) p")30 #expect(close(r.correlation, c.double("r"), rtol: 1e-12) && close(r.rSquared, c.double("rSquared"), rtol: 1e-12))31 #expect(close(r.adjustedRSquared, c.double("adjustedRSquared"), rtol: 1e-11) && close(r.residualStandardError, c.double("sigma"), rtol: 1e-10))32 #expect(close(r.residualSumOfSquares, c.double("rss"), rtol: 1e-10))33 #expect(close(r.fStatistic, c.double("f"), rtol: 1e-9) && close(r.fPValue, c.double("fp"), rtol: 1e-8, atol: 1e-300))34 #expect(close(r.slope.lower, c.double("slopeLower"), rtol: 1e-10) && close(r.slope.upper, c.double("slopeUpper"), rtol: 1e-10))35 #expect(close(r.intercept.lower, c.double("interceptLower"), rtol: 1e-9) && close(r.intercept.upper, c.double("interceptUpper"), rtol: 1e-9))36 let p = try r.predict(at: c.double("x0"))37 #expect(close(p.fitted, c.double("fitted"), rtol: 1e-12))38 #expect(close(p.meanLower, c.double("meanLower"), rtol: 1e-10) && close(p.meanUpper, c.double("meanUpper"), rtol: 1e-10), "\(name) mean CI")39 #expect(close(p.predictionLower, c.double("predLower"), rtol: 1e-10) && close(p.predictionUpper, c.double("predUpper"), rtol: 1e-10), "\(name) PI")40 #expect(p.predictionUpper - p.predictionLower > p.meanUpper - p.meanLower)41 #expect(r.verification.checks.contains { $0.name == "Normal equations" && $0.status == .passed })42 // The closed-form path and the QR path are two independent implementations: they must agree.43 let qr = try LinearRegression.multiple(y: y, predictors: [x], level: c.double("level"))44 #expect(close(qr.coefficients[1].estimate, r.slope.estimate, rtol: 1e-11) && close(qr.coefficients[0].estimate, r.intercept.estimate, rtol: 1e-10))45 #expect(close(qr.coefficients[1].standardError, r.slope.standardError, rtol: 1e-9) && close(qr.residualStandardError, r.residualStandardError, rtol: 1e-9))46 let qp = try qr.predict(at: [c.double("x0")])47 #expect(close(qp.predictionLower, p.predictionLower, rtol: 1e-9) && close(qp.meanUpper, p.meanUpper, rtol: 1e-9))48 }49 }5051 /// Measured worst coefficient error against the 60-digit solution: 9e-12 on Longley (numpy.linalg.lstsq: 1.0e-11),52 /// 1e-13 on the nearly collinear set, 6e-15 otherwise.53 static func tolerance(_ name: String) -> Double { name == "longley" ? 1e-10 : 1e-11 }5455 @Test func multipleRegressionMatchesStatsmodelsAndExactSolve() throws {56 for c in try Self.cases("multiple") {57 let name = c.string("name"), tol = Self.tolerance(name)58 let hasIntercept = (c["intercept"] as? Bool) == true59 let r = try LinearRegression.multiple(y: c.doubles("y"), predictors: c.matrix("predictors"), intercept: hasIntercept, level: c.double("level"))60 let beta = c.doubles("coefficients")61 #expect(r.parameterCount == beta.count && r.hasIntercept == hasIntercept && r.count == c.doubles("y").count)62 #expect(allClose(r.coefficients.map(\.estimate), beta, rtol: tol), "\(name) β: \(r.coefficients.map(\.estimate)) vs \(beta)")63 #expect(allClose(r.coefficients.map(\.standardError), c.doubles("standardErrors"), rtol: tol), "\(name) SE")64 #expect(allClose(r.coefficients.map(\.tStatistic), c.doubles("t"), rtol: tol), "\(name) t")65 #expect(allClose(r.coefficients.map(\.pValue), c.doubles("p"), rtol: max(tol, 1e-7), atol: 1e-300), "\(name) p")66 #expect(allClose(r.coefficients.map(\.lower), c.doubles("ciLower"), rtol: max(tol, 1e-8), atol: 1e-9) && allClose(r.coefficients.map(\.upper), c.doubles("ciUpper"), rtol: max(tol, 1e-8), atol: 1e-9), "\(name) CI")67 #expect(close(r.residualSumOfSquares, c.double("rss"), rtol: tol) && close(r.residualStandardError, c.double("sigma"), rtol: tol), "\(name) RSS")68 #expect(close(r.rSquared, c.double("rSquared"), rtol: 1e-10) && close(r.adjustedRSquared, c.double("adjustedRSquared"), rtol: 1e-10), "\(name) R²")69 #expect(try close(try #require(r.fStatistic), c.double("f"), rtol: tol) && close(try #require(r.fPValue), c.double("fp"), rtol: max(tol, 1e-7), atol: 1e-300), "\(name) F")70 #expect(allClose(r.fitted, c.doubles("fitted"), rtol: 1e-9) && allClose(r.residuals, c.doubles("residuals"), rtol: 1e-6, atol: 1e-7), "\(name) fitted")71 #expect(allClose(r.leverage, c.doubles("leverage"), rtol: 1e-8), "\(name) leverage")72 #expect(allClose(r.standardizedResiduals, c.doubles("standardized"), rtol: 1e-6, atol: 1e-8), "\(name) standardized")73 #expect(allClose(r.studentizedResiduals, c.doubles("studentized"), rtol: 1e-6, atol: 1e-8), "\(name) studentized")74 #expect(allClose(r.cooksDistance, c.doubles("cooks"), rtol: 1e-6, atol: 1e-10), "\(name) Cook")75 #expect(close(r.durbinWatson, c.double("durbinWatson"), rtol: 1e-7), "\(name) DW")76 #expect(close(r.logLikelihood, c.double("logLikelihood"), rtol: 1e-9) && close(r.aic, c.double("aic"), rtol: 1e-9) && close(r.bic, c.double("bic"), rtol: 1e-9), "\(name) IC")77 #expect(allClose(r.varianceInflationFactors, c.doubles("vif"), rtol: max(tol, 1e-8)), "\(name) VIF \(r.varianceInflationFactors) vs \(c.doubles("vif"))")78 #expect(close(r.leverage.reduce(0, +), Double(beta.count), rtol: 1e-9))79 let p = try r.predict(at: c.doubles("x0"))80 let e = c.dict("prediction")81 #expect(close(p.fitted, e.double("fitted"), rtol: max(tol, 1e-9)) && close(p.standardErrorOfMean, e.double("seMean"), rtol: max(tol, 1e-8)), "\(name) prediction")82 #expect(close(p.meanLower, e.double("meanLower"), rtol: max(tol, 1e-8)) && close(p.meanUpper, e.double("meanUpper"), rtol: max(tol, 1e-8)))83 #expect(close(p.predictionLower, e.double("predLower"), rtol: max(tol, 1e-8)) && close(p.predictionUpper, e.double("predUpper"), rtol: max(tol, 1e-8)))84 #expect(close(try r.value(at: c.doubles("x0")), p.fitted, rtol: 1e-15))85 #expect(r.verification.checks.contains { $0.name.hasPrefix("Normal equations") })86 // Our QR is at least as close to the exact solution as numpy.linalg.lstsq (SVD) on every data set.87 let ours = zip(r.coefficients.map(\.estimate), beta).map { relativeError($0, $1) }.max()!88 let numpy = zip(c.doubles("numpyLstsq"), beta).map { relativeError($0, $1) }.max()!89 #expect(ours <= max(10 * numpy, 1e-12), "\(name): our worst coefficient error \(ours), numpy lstsq \(numpy)")90 }91 }9293 @Test func longleyReproducesTheCertifiedValues() throws {94 // The reference coefficients come from a 60-digit solve of the data as shipped by statsmodels. They coincide with the NIST95 // StRD certified values for Longley to all 15 printed digits (UNEMP and ARMED are in units 100× larger in that copy).96 let c = try #require(try Self.cases("multiple").first { $0.string("name") == "longley" })97 let r = try LinearRegression.multiple(y: c.doubles("y"), predictors: c.matrix("predictors"), names: c["predictorNames"] as? [String])98 let certified = [-3482258.63459582, 15.0618722713733, -0.358191792925910e-01, -0.202022980381683e-01 * 100, -0.103322686717359e-01 * 100,99 -0.511041056535807e-01, 1829.15146461355]100 #expect(allClose(c.doubles("coefficients"), certified, rtol: 5e-14))101 let digits = zip(r.coefficients.map(\.estimate), certified).map { -log10(max(relativeError($0, $1), 1e-16)) }102 #expect(digits.min()! >= 10, "matching digits per coefficient: \(digits)")103 #expect(close(r.residualStandardError, 304.854073561965, rtol: 1e-9) && close(r.rSquared, 0.995479004577296, rtol: 1e-12))104 #expect(r.coefficients.map(\.name) == ["intercept", "GNPDEFL", "GNP", "UNEMP", "ARMED", "POP", "YEAR"])105 // The ill-conditioning is detected and reported, not hidden.106 #expect(r.verification.warnings.contains { $0.name == "Conditioning" } && r.verification.warnings.contains { $0.name == "Multicollinearity" })107 #expect(r.varianceInflationFactors.max()! > 1000)108 }109110 @Test func curveFitsMatchNumpyAndScipy() throws {111 for c in try Self.cases("curve") {112 let x = c.doubles("x"), y = c.doubles("y"), name = c.string("model")113 let model: CurveModel = switch name {114 case "linear": .linear115 case "quadratic": .quadratic116 case "cubic": .cubic117 case "exponential": .exponential118 case "logarithmic": .logarithmic119 case "power": .power120 default: .polynomial(degree: c.int("degree"))121 }122 let r = try CurveFit.fit(x: x, y: y, model: model)123 let tol = name == "polynomial5" ? 1e-6 : 1e-9124 #expect(allClose(r.coefficients, c.doubles("coefficients"), rtol: tol, atol: 1e-9), "\(name): \(r.coefficients) vs \(c.doubles("coefficients"))")125 #expect(close(r.rSquared, c.double("rSquared"), rtol: 1e-9), "\(name) R² \(r.rSquared) vs \(c.double("rSquared"))")126 #expect(close(r.rSquaredOriginalScale, c.double("rSquaredOriginal"), rtol: 1e-9), "\(name) R² original")127 let transformed = name == "exponential" || name == "power"128 #expect(r.rSquaredScale == (transformed ? .transformed : .original))129 #expect(r.verification.warnings.contains { $0.name == "R² scale" } == transformed)130 #expect(r.explanation.steps.contains { $0.label == (transformed ? "R² (on ln y)" : "R²") })131 #expect(r.regression != nil && r.count == x.count)132 // value(at:) evaluates the reported equation.133 let x0 = x[3]134 let expected: Double = switch name {135 case "exponential": r.coefficients[0] * pow(r.coefficients[1], x0)136 case "power": r.coefficients[0] * pow(x0, r.coefficients[1])137 case "logarithmic": r.coefficients[0] + r.coefficients[1] * log(x0)138 default: r.coefficients.enumerated().reduce(0) { $0 + $1.element * pow(x0, Double($1.offset)) }139 }140 #expect(close(try r.value(at: x0), expected, rtol: 1e-12))141 }142 }143}144145@Suite struct RegressionBehaviourTests {146 @Test func exactLineAndExactInterpolation() throws {147 // y = 2x + 1 exactly: slope and intercept exact, perfect-fit warning, no NaN anywhere.148 let x = [1.0, 2, 3, 4, 5], y = [3.0, 5, 7, 9, 11]149 let r = try LinearRegression.simple(x: x, y: y)150 #expect(r.slope.estimate == 2 && r.intercept.estimate == 1 && r.rSquared == 1 && r.correlation == 1 && r.residualStandardError == 0)151 #expect(r.slope.tStatistic == .infinity && r.slope.pValue == 0 && r.intercept.pValue == 0)152 #expect(r.verification.warnings.contains { $0.name == "Perfect fit" })153 #expect(r.value(at: 10) == 21)154 let m = try LinearRegression.multiple(y: y, predictors: [x])155 #expect(close(m.coefficients[1].estimate, 2, rtol: 1e-13) && m.standardizedResiduals.allSatisfy { $0.isFinite } && m.cooksDistance.allSatisfy { $0.isFinite })156 // Three points determine a parabola: coefficients only, no inference, with an explicit warning (TI QuadReg accepts this too).157 let q = try CurveFit.fit(x: [0, 1, 2], y: [1, 3, 9], model: .quadratic)158 #expect(allClose(q.coefficients, [1, 0, 2], rtol: 1e-12, atol: 1e-12) && q.regression == nil && q.rSquared == 1)159 #expect(q.verification.warnings.contains { $0.name == "Exact interpolation" })160 #expect(q.equation.hasPrefix("y = 1 + "))161 // Exact exponential and power laws are recovered.162 let e = try CurveFit.fit(x: [0, 1, 2, 3, 4], y: [3, 6, 12, 24, 48], model: .exponential)163 #expect(allClose(e.coefficients, [3, 2], rtol: 1e-12))164 let p = try CurveFit.fit(x: [1, 2, 3, 4, 5], y: [2, 8, 18, 32, 50], model: .power)165 #expect(allClose(p.coefficients, [2, 2], rtol: 1e-12))166 let l = try CurveFit.fit(x: [1, M_E, M_E * M_E, 20], y: [5, 8, 11, 5 + 3 * log(20)], model: .logarithmic)167 #expect(allClose(l.coefficients, [5, 3], rtol: 1e-12))168 }169170 @Test func rankDeficiencyIsDetectedAndExplained() throws {171 let x1 = [1.0, 2, 3, 4, 5, 6, 7, 8], x2 = [2.0, 1, 4, 3, 6, 5, 8, 7]172 let y = [3.1, 2.9, 7.2, 6.8, 11.1, 10.9, 15.2, 14.8]173 let x3 = zip(x1, x2).map { 2 * $0 - 3 * $1 } // exact linear combination174 do {175 _ = try LinearRegression.multiple(y: y, predictors: [x1, x2, x3], names: ["a", "b", "c"])176 Issue.record("a collinear design must throw")177 } catch let error as StatisticsError {178 guard case .singularDesignMatrix(let message) = error else { Issue.record("wrong error \(error)"); return }179 #expect(message.contains("rank deficient") && message.contains("rank 3 of 4"))180 }181 #expect(throws: StatisticsError.self) { try LinearRegression.multiple(y: y, predictors: [x1, x1]) } // duplicated predictor182 #expect(throws: StatisticsError.self) { try LinearRegression.multiple(y: y, predictors: [[Double](repeating: 4, count: 8)]) } // constant = intercept183 #expect(throws: StatisticsError.self) { try LinearRegression.multiple(y: y, predictors: [[Double](repeating: 0, count: 8)], intercept: false) }184 #expect(throws: StatisticsError.self) { try LinearRegression.simple(x: [2, 2, 2, 2], y: [1, 2, 3, 4]) }185 #expect(throws: StatisticsError.self) { try CurveFit.fit(x: [1, 1, 1, 1], y: [1, 2, 3, 4], model: .quadratic) }186 // Nearly — but not exactly — collinear predictors are fitted, with a conditioning warning.187 let near = zip(x1, x2).map { 2 * $0 - 3 * $1 + 1e-7 * sin($0) }188 let fit = try LinearRegression.multiple(y: y, predictors: [x1, x2, near])189 #expect(fit.verification.warnings.contains { $0.name == "Conditioning" })190 }191192 @Test func errorPaths() {193 #expect(throws: StatisticsError.emptyData) { try LinearRegression.simple(x: [], y: []) }194 #expect(throws: StatisticsError.insufficientSample(required: 3, actual: 2)) { try LinearRegression.simple(x: [1, 2], y: [1, 2]) }195 #expect(throws: StatisticsError.self) { try LinearRegression.simple(x: [1, 2, 3], y: [1, 2]) }196 #expect(throws: StatisticsError.self) { try LinearRegression.simple(x: [1, 2, .nan], y: [1, 2, 3]) }197 #expect(throws: StatisticsError.self) { try LinearRegression.simple(x: [1, 2, 3], y: [1, 2, 4], level: 1) }198 #expect(throws: StatisticsError.insufficientSample(required: 4, actual: 3)) { try LinearRegression.multiple(y: [1, 2, 3], predictors: [[1, 2, 4], [3, 1, 2]]) }199 #expect(throws: StatisticsError.self) { try LinearRegression.multiple(y: [1, 2, 3, 4], predictors: [[1, 2, 4]]) }200 #expect(throws: StatisticsError.self) { try LinearRegression.multiple(y: [1, 2, 3, 4], predictors: [], intercept: false) }201 #expect(throws: StatisticsError.self) { try LinearRegression.multiple(y: [1, 2, 3, 4], predictors: [[1, 2, 4, 9]], names: ["a", "b"]) }202 #expect(throws: StatisticsError.self) { try CurveFit.fit(x: [1, 2, 3], y: [1, -2, 3], model: .exponential) }203 #expect(throws: StatisticsError.self) { try CurveFit.fit(x: [0, 2, 3], y: [1, 2, 3], model: .logarithmic) }204 #expect(throws: StatisticsError.self) { try CurveFit.fit(x: [1, 2, 3], y: [1, 0, 3], model: .power) }205 #expect(throws: StatisticsError.self) { try CurveFit.fit(x: [1, 2, 3], y: [1, 2, 3], model: .cubic) }206 #expect(throws: StatisticsError.self) { try CurveFit.fit(x: [1, 2, 3], y: [1, 2, 3], model: .polynomial(degree: 0)) }207 #expect(throws: StatisticsError.self) { try CurveFit.fit(x: [1, 2, 3], y: [1, 2, 3], model: .polynomial(degree: 13)) }208 }209210 @Test func predictionValidation() throws {211 let r = try LinearRegression.multiple(y: [1, 3, 2, 5, 4, 6], predictors: [[1, 2, 3, 4, 5, 6], [2, 1, 4, 3, 6, 5.5]])212 #expect(throws: StatisticsError.self) { try r.predict(at: [1]) }213 #expect(throws: StatisticsError.self) { try r.predict(at: [1, .nan]) }214 #expect(throws: StatisticsError.self) { try r.predict(at: [1, 2], level: 0) }215 let wide = try r.predict(at: [3, 3], level: 0.99), narrow = try r.predict(at: [3, 3], level: 0.8)216 #expect(wide.meanUpper - wide.meanLower > narrow.meanUpper - narrow.meanLower && wide.level == 0.99)217 // An intercept-only model has no overall F test.218 let mean = try LinearRegression.multiple(y: [1, 3, 2, 5], predictors: [])219 #expect(mean.fStatistic == nil && mean.coefficients[0].estimate == 2.75 && mean.varianceInflationFactors.isEmpty)220 let log = try CurveFit.fit(x: [1, 2, 3, 4], y: [1, 2, 2.5, 3], model: .logarithmic)221 #expect(throws: StatisticsError.self) { try log.value(at: -1) }222 }223}224