Repository Public
1import Foundation2import SheetCore34/// Estimator of the coefficient covariance matrix of a linear model.5public enum CovarianceType: Codable, Sendable, Hashable {6 /// Classical σ²(XᵀX)⁻¹ with σ² = SSR / (n − k).7 case nonrobust8 /// White's heteroskedasticity-consistent estimator, weights eᵢ².9 case hc010 /// HC0 × n / (n − k).11 case hc112 /// Weights eᵢ² / (1 − hᵢ).13 case hc214 /// Weights eᵢ² / (1 − hᵢ)² (close to the jackknife).15 case hc316 /// Newey–West HAC with the Bartlett kernel wₗ = 1 − l/(maxLags + 1) on the scores xₜeₜ,17 /// observations in time order. `smallSample` multiplies by n / (n − k)18 /// (statsmodels `cov_type='HAC'`, `use_correction`).19 case hac(maxLags: Int, smallSample: Bool)20 /// Cluster-robust (Liang–Zeger) covariance: sums of scores within each group. `smallSample`21 /// multiplies by G/(G − 1) · (n − 1)/(n − k) (statsmodels `cov_type='cluster'` default, Stata `regress, vce(cluster)`).22 case cluster(groups: [Int], smallSample: Bool)2324 public var label: String {25 switch self {26 case .nonrobust: "nonrobust"27 case .hc0: "HC0"28 case .hc1: "HC1"29 case .hc2: "HC2"30 case .hc3: "HC3"31 case .hac(let l, let s): "HAC (Newey–West, Bartlett, \(l) lags\(s ? ", small-sample" : ""))"32 case .cluster(let g, let s): "cluster (\(Set(g).count) groups\(s ? ", small-sample" : ""))"33 }34 }35 var isRobust: Bool { if case .nonrobust = self { false } else { true } }36}3738/// A fitted linear regression (OLS or WLS).39///40/// Conventions (identical to statsmodels `OLS`/`WLS`):41/// - `dfResidual` = n − k (k counts the intercept); `sigma2` = SSR / (n − k).42/// - R² is centred when the model has a constant, uncentred otherwise; adjusted R² = 1 − (n − c)/(n − k)·(1 − R²), c = 1 with a constant.43/// - `logLikelihood` is the Gaussian log-likelihood concentrated over σ² (σ̂² = SSR/n):44/// −n/2·(ln 2π + ln(SSR/n) + 1) (+ ½ Σ ln wᵢ for WLS); AIC = −2ℓ + 2k, BIC = −2ℓ + k ln n (σ² is not counted as a parameter).45/// - With the classical covariance, inference uses Student t(n − k) and the F test is the usual46/// (ESS/df_model)/(SSR/df_resid). With a robust covariance, p-values and intervals use the normal47/// distribution unless `useT` was requested (then t with n − k df, or G − 1 for clusters) and the48/// F statistic is the Wald test of all slopes = 0 divided by its df, referred to F(df_model, df_inference).49/// - All p-values of coefficients are two-sided; the F p-value is upper-tail.50public struct LinearModelResult: Codable, Sendable, Hashable {51 /// "OLS" or "WLS".52 public var method: String53 public var dependentName: String54 /// Coefficient names, intercept ("const") first when present.55 public var names: [String]56 /// True when `fit` added the constant column.57 public var hasIntercept: Bool58 /// True when the design contains a constant column (added by `fit` or supplied among the regressors);59 /// decides between centred and uncentred R².60 public var hasConstant: Bool61 public var nobs: Int62 /// Number of regressors excluding the constant.63 public var dfModel: Int64 public var dfResidual: Int6566 public var coefficients: [Double]67 public var standardErrors: [Double]68 /// t (or z) statistics.69 public var statistics: [Double]70 /// Two-sided p-values.71 public var pValues: [Double]72 public var ciLower: [Double]73 public var ciUpper: [Double]74 public var confidenceLevel: Double75 /// Covariance matrix of the coefficients.76 public var covariance: Matrix77 public var covarianceType: CovarianceType78 /// True when inference uses Student t, false for the normal distribution.79 public var usesT: Bool80 /// Degrees of freedom used for t and for the denominator of F.81 public var dfInference: Double8283 public var rSquared: Double84 public var adjustedRSquared: Double85 /// nil when the model has no regressor besides the constant.86 public var fStatistic: Double?87 /// Upper-tail p-value of `fStatistic`.88 public var fPValue: Double?89 /// SSR / (n − k).90 public var sigma2: Double91 public var residualStandardError: Double { sigma2.squareRoot() }92 /// Sum of squared (weighted, for WLS) residuals.93 public var ssr: Double94 /// Total sum of squares (centred when the model has a constant; weighted for WLS).95 public var tss: Double96 public var logLikelihood: Double97 public var aic: Double98 public var bic: Double99100 public var fitted: [Double]101 /// y − ŷ in the original units (unweighted, also for WLS).102 public var residuals: [Double]103 /// Diagonal of the hat matrix (of the weighted model for WLS).104 public var leverage: [Double]105 /// Internally studentised residuals eᵢ / (σ̂ √(1 − hᵢ)). An observation with leverage 1 is fitted106 /// exactly (its residual is 0 and the ratio is undefined): it is reported as 0, as is its Cook's distance.107 public var studentizedResiduals: [Double]108 /// Cook's distance Dᵢ = rᵢ² hᵢ / (k (1 − hᵢ)), rᵢ the internally studentised residual.109 public var cooksDistance: [Double]110 /// Variance inflation factors 1/(1 − R²ⱼ), regressing each column on all the others111 /// (constant included); nil for the constant itself and in models without a constant column.112 public var vif: [Double?]113 /// Ratio of the largest to the smallest singular value of the design matrix (unscaled columns, as statsmodels).114 public var conditionNumber: Double115116 /// Design matrix actually used (constant column first when `hasIntercept`).117 public var design: Matrix118 public var y: [Double]119 /// Observation weights (WLS), nil for OLS.120 public var weights: [Double]?121122 /// Predictions for new rows of regressors (without the constant column; it is added when the model has one).123 public func predict(_ x: Matrix) throws -> [Double] {124 let expect = names.count - (hasIntercept ? 1 : 0)125 guard x.cols == expect else { throw StatError.invalidInput("expected \(expect) regressors, got \(x.cols)") }126 return (0..<x.rows).map { r in127 var s = hasIntercept ? coefficients[0] : 0128 for c in 0..<x.cols { s += x[r, c] * coefficients[c + (hasIntercept ? 1 : 0)] }129 return s130 }131 }132133 /// Design and residuals of the transformed (√w-weighted) model on which the algebra is done.134 var whitened: (x: Matrix, e: [Double]) {135 guard let w = weights else { return (design, residuals) }136 var x = design137 var e = residuals138 for i in 0..<x.rows {139 let s = w[i].squareRoot()140 for j in 0..<x.cols { x[i, j] *= s }141 e[i] *= s142 }143 return (x, e)144 }145}146147public enum LinearRegression {148 /// Ordinary (or, with `weights`, weighted) least squares by Householder QR (LAPACK `dgeqrf`).149 ///150 /// - Parameters:151 /// - y: dependent variable.152 /// - x: regressors, one column per variable, WITHOUT a constant column.153 /// - names: one name per column of `x` (default x1, x2, …).154 /// - intercept: add a constant ("const") as the first column.155 /// - weights: WLS weights wᵢ > 0, proportional to the inverse error variances (statsmodels `WLS`).156 /// - covariance: covariance estimator, see `CovarianceType`.157 /// - useT: nil = Student t for the classical covariance and normal for robust ones (statsmodels defaults).158 /// - Throws: `StatError.insufficientData` when n ≤ k, `StatError.singular` when the design is rank deficient159 /// (perfect collinearity, a constant regressor next to the intercept, the dummy-variable trap).160 public static func fit(y: [Double], x: Matrix, names: [String]? = nil, intercept: Bool = true, weights: [Double]? = nil,161 covariance: CovarianceType = .nonrobust, confidenceLevel: Double = 0.95, useT: Bool? = nil,162 dependentName: String = "y") throws -> LinearModelResult {163 let n = y.count164 guard x.rows == n else { throw StatError.invalidInput("y has \(n) observations but X has \(x.rows) rows") }165 guard confidenceLevel > 0, confidenceLevel < 1 else { throw StatError.invalidInput("confidence level must be in (0, 1)") }166 if let names, names.count != x.cols { throw StatError.invalidInput("\(names.count) names for \(x.cols) regressors") }167 if let w = weights {168 guard w.count == n else { throw StatError.invalidInput("one weight per observation is required") }169 guard w.allSatisfy({ $0 > 0 && $0.isFinite }) else { throw StatError.invalidInput("weights must be positive and finite") }170 }171 var colNames = names ?? (0..<x.cols).map { "x\($0 + 1)" }172 var design = x173 if intercept {174 var d = Matrix(rows: n, cols: x.cols + 1)175 for i in 0..<n {176 d[i, 0] = 1177 for j in 0..<x.cols { d[i, j + 1] = x[i, j] }178 }179 design = d180 colNames.insert("const", at: 0)181 }182 let k = design.cols183 guard k > 0 else { throw StatError.invalidInput("the model has no regressors") }184 guard n > k else { throw StatError.insufficientData("\(n) observations for \(k) coefficients: at least \(k + 1) are needed") }185186 // transformed model187 var xw = design, yw = y188 if let w = weights {189 for i in 0..<n {190 let s = w[i].squareRoot()191 for j in 0..<k { xw[i, j] *= s }192 yw[i] *= s193 }194 }195 let core = try RegressionMath.solve(xw, yw, names: colNames)196 let hasConst = intercept || (0..<k).contains { RegressionMath.isConstantColumn(design, $0) }197 let dfResid = n - k198 let dfModel = k - (hasConst ? 1 : 0)199 let ssr = core.ssr200 let sigma2 = ssr / Double(dfResid)201 let tss: Double202 if hasConst {203 if let w = weights {204 let sw = Descriptive.sum(w)205 let mean = zip(w, y).reduce(0) { $0 + $1.0 * $1.1 } / sw206 tss = zip(w, y).reduce(0) { $0 + $1.0 * ($1.1 - mean) * ($1.1 - mean) }207 } else { tss = RegressionMath.centeredSS(y) }208 } else { tss = RegressionMath.uncenteredSS(yw) }209 guard tss > 0 else { throw StatError.invalidInput("the dependent variable is constant: there is nothing to explain") }210 let r2 = 1 - ssr / tss211 let adj = 1 - Double(n - (hasConst ? 1 : 0)) / Double(dfResid) * (1 - r2)212 let nd = Double(n)213 var llf = -nd / 2 * (log(2 * Double.pi) + log(ssr / nd) + 1)214 if let w = weights { llf += 0.5 * w.reduce(0) { $0 + log($1) } }215216 let fitted = design * core.beta217 let resid = zip(y, fitted).map { $0 - $1 }218 let lev = core.leverage219 var student = [Double](repeating: 0, count: n), cooks = [Double](repeating: 0, count: n)220 for i in 0..<n {221 let one = 1 - lev[i]222 guard one > 1e-12, sigma2 > 0 else { continue }223 let r = core.residuals[i] / (sigma2 * one).squareRoot()224 student[i] = r225 cooks[i] = r * r * lev[i] / (Double(k) * one)226 }227 // VIF228 var vif = [Double?](repeating: nil, count: k)229 if hasConst, k >= 2 {230 for j in 0..<k where !RegressionMath.isConstantColumn(design, j) {231 var others = Matrix(rows: n, cols: k - 1)232 var target = [Double](repeating: 0, count: n)233 for i in 0..<n {234 var c = 0235 for jj in 0..<k {236 if jj == j { target[i] = design[i, jj] } else { others[i, c] = design[i, jj]; c += 1 }237 }238 }239 if let aux = try? RegressionMath.solve(others, target) {240 let t = RegressionMath.centeredSS(target)241 vif[j] = aux.ssr > 0 ? t / aux.ssr : .infinity242 }243 }244 }245 let cond = (try? LinearAlgebra.conditionNumber(design)) ?? .nan246247 var result = LinearModelResult(248 method: weights == nil ? "OLS" : "WLS", dependentName: dependentName, names: colNames, hasIntercept: intercept, hasConstant: hasConst,249 nobs: n, dfModel: dfModel, dfResidual: dfResid, coefficients: core.beta, standardErrors: [], statistics: [], pValues: [],250 ciLower: [], ciUpper: [], confidenceLevel: confidenceLevel, covariance: Matrix(rows: k, cols: k), covarianceType: .nonrobust,251 usesT: true, dfInference: Double(dfResid), rSquared: r2, adjustedRSquared: adj, fStatistic: nil, fPValue: nil,252 sigma2: sigma2, ssr: ssr, tss: tss, logLikelihood: llf, aic: -2 * llf + 2 * Double(k), bic: -2 * llf + log(nd) * Double(k),253 fitted: fitted, residuals: resid, leverage: lev, studentizedResiduals: student, cooksDistance: cooks, vif: vif,254 conditionNumber: cond, design: design, y: y, weights: weights)255 try applyCovariance(&result, covariance, useT: useT, xtxInverse: core.xtxInverse)256 return result257 }258259 /// Convenience overload taking the regressors as columns.260 public static func fit(y: [Double], columns: [[Double]], names: [String]? = nil, intercept: Bool = true, weights: [Double]? = nil,261 covariance: CovarianceType = .nonrobust, confidenceLevel: Double = 0.95, useT: Bool? = nil,262 dependentName: String = "y") throws -> LinearModelResult {263 guard columns.allSatisfy({ $0.count == y.count }) else { throw StatError.invalidInput("every regressor needs one value per observation") }264 let x = columns.isEmpty ? Matrix(rows: y.count, cols: 0) : Matrix(columns: columns)265 return try fit(y: y, x: x, names: names, intercept: intercept, weights: weights, covariance: covariance,266 confidenceLevel: confidenceLevel, useT: useT, dependentName: dependentName)267 }268269 /// Covariance matrix of the coefficients of a fitted model under another estimator.270 public static func covarianceMatrix(_ model: LinearModelResult, _ type: CovarianceType) throws -> Matrix {271 let (x, e) = model.whitened272 let n = x.rows, k = x.cols273 let yw = zip(model.y, model.weights ?? Array(repeating: 1, count: n)).map { $0 * $1.squareRoot() }274 let core = try RegressionMath.solve(x, yw, names: model.names)275 return try covarianceMatrix(type, x: x, e: e, leverage: model.leverage, sigma2: model.sigma2, xtxInverse: core.xtxInverse, n: n, k: k)276 }277278 static func covarianceMatrix(_ type: CovarianceType, x: Matrix, e: [Double], leverage h: [Double], sigma2: Double,279 xtxInverse bread: Matrix, n: Int, k: Int) throws -> Matrix {280 switch type {281 case .nonrobust:282 return sigma2 * bread283 case .hc0:284 return RegressionMath.sandwich(bread, RegressionMath.weightedGram(x, e.map { $0 * $0 }))285 case .hc1:286 return Double(n) / Double(n - k) * RegressionMath.sandwich(bread, RegressionMath.weightedGram(x, e.map { $0 * $0 }))287 case .hc2:288 return RegressionMath.sandwich(bread, RegressionMath.weightedGram(x, zip(e, h).map { $0 * $0 / (1 - $1) }))289 case .hc3:290 return RegressionMath.sandwich(bread, RegressionMath.weightedGram(x, zip(e, h).map { $0 * $0 / ((1 - $1) * (1 - $1)) }))291 case .hac(let maxLags, let small):292 guard maxLags >= 0, maxLags < n else { throw StatError.invalidInput("HAC lags must be between 0 and n − 1") }293 var s = RegressionMath.weightedGram(x, e.map { $0 * $0 })294 for lag in 0..<maxLags {295 let l = lag + 1296 let w = 1 - Double(l) / Double(maxLags + 1)297 var g = Matrix(rows: k, cols: k)298 for t in l..<n {299 let et = e[t] * e[t - l]300 for a in 0..<k { let xa = x[t, a] * et; for b in 0..<k { g[a, b] += xa * x[t - l, b] } }301 }302 s = s + w * (g + g.transposed)303 }304 let v = RegressionMath.sandwich(bread, s)305 return small ? Double(n) / Double(n - k) * v : v306 case .cluster(let groups, let small):307 guard groups.count == n else { throw StatError.invalidInput("one cluster label per observation is required") }308 var sums = [Int: [Double]]()309 for i in 0..<n {310 var acc = sums[groups[i]] ?? [Double](repeating: 0, count: k)311 for j in 0..<k { acc[j] += x[i, j] * e[i] }312 sums[groups[i]] = acc313 }314 let g = sums.count315 guard g >= 2 else { throw StatError.insufficientData("cluster-robust covariance needs at least 2 clusters") }316 var s = Matrix(rows: k, cols: k)317 for v in sums.values { for a in 0..<k { for b in 0..<k { s[a, b] += v[a] * v[b] } } }318 let cov = RegressionMath.sandwich(bread, s)319 guard small else { return cov }320 return Double(g) / Double(g - 1) * Double(n - 1) / Double(n - k) * cov321 }322 }323324 /// Returns the same fit with standard errors, tests and intervals recomputed under another covariance estimator.325 public static func reestimate(_ model: LinearModelResult, covariance: CovarianceType, useT: Bool? = nil) throws -> LinearModelResult {326 var out = model327 let (x, _) = model.whitened328 let yw = zip(model.y, model.weights ?? Array(repeating: 1, count: model.nobs)).map { $0 * $1.squareRoot() }329 let core = try RegressionMath.solve(x, yw, names: model.names)330 try applyCovariance(&out, covariance, useT: useT, xtxInverse: core.xtxInverse)331 return out332 }333334 private static func applyCovariance(_ r: inout LinearModelResult, _ type: CovarianceType, useT: Bool?, xtxInverse: Matrix) throws {335 let (x, e) = r.whitened336 let n = r.nobs, k = r.names.count337 let cov = try covarianceMatrix(type, x: x, e: e, leverage: r.leverage, sigma2: r.sigma2, xtxInverse: xtxInverse, n: n, k: k)338 let t = useT ?? !type.isRobust339 var df = Double(r.dfResidual)340 if case .cluster(let g, _) = type { df = Double(Set(g).count - 1) }341 r.covariance = cov342 r.covarianceType = type343 r.usesT = t344 r.dfInference = df345 let se = (0..<k).map { cov[$0, $0].squareRoot() }346 let stat = (0..<k).map { r.coefficients[$0] / se[$0] }347 let tDist = StudentTDistribution(df: df)348 let alpha = 1 - r.confidenceLevel349 let crit = t ? tDist.isf(alpha / 2) : NormalDistribution.standard.isf(alpha / 2)350 r.standardErrors = se351 r.statistics = stat352 r.pValues = stat.map { s in353 return t ? 2 * tDist.sf(abs(s)) : 2 * NormalDistribution.standard.sf(abs(s))354 }355 r.ciLower = (0..<k).map { r.coefficients[$0] - crit * se[$0] }356 r.ciUpper = (0..<k).map { r.coefficients[$0] + crit * se[$0] }357 // overall F358 r.fStatistic = nil; r.fPValue = nil359 var fStat: Double360 guard r.dfModel > 0 else { return }361 if case .nonrobust = type {362 let ess = r.tss - r.ssr363 fStat = (ess / Double(r.dfModel)) / r.sigma2364 } else {365 let slopes = r.hasConstant ? (0..<k).filter { !RegressionMath.isConstantColumn(r.design, $0) } : Array(0..<k)366 guard !slopes.isEmpty, let w = try? RegressionMath.wald(r.coefficients, cov, slopes) else { return }367 fStat = w / Double(slopes.count)368 }369 guard fStat.isFinite else { return }370 r.fStatistic = fStat371 r.fPValue = FisherFDistribution(d1: Double(r.dfModel), d2: df).sf(fStat)372 }373}374