SPB Git

Repository Public

LinearModel.swift 19.4 KB 374 lines swift

1import Foundation2import SheetCore34/// Estimator of the coefficient covariance matrix of a linear model.5public enum CovarianceType: Codable, Sendable, Hashable {6    /// Classical σ²(XᵀX)⁻¹ with σ² = SSR / (n − k).7    case nonrobust8    /// White's heteroskedasticity-consistent estimator, weights eᵢ².9    case hc010    /// HC0 × n / (n − k).11    case hc112    /// Weights eᵢ² / (1 − hᵢ).13    case hc214    /// Weights eᵢ² / (1 − hᵢ)² (close to the jackknife).15    case hc316    /// Newey–West HAC with the Bartlett kernel wₗ = 1 − l/(maxLags + 1) on the scores xₜeₜ,17    /// observations in time order. `smallSample` multiplies by n / (n − k)18    /// (statsmodels `cov_type='HAC'`, `use_correction`).19    case hac(maxLags: Int, smallSample: Bool)20    /// Cluster-robust (Liang–Zeger) covariance: sums of scores within each group. `smallSample`21    /// multiplies by G/(G − 1) · (n − 1)/(n − k) (statsmodels `cov_type='cluster'` default, Stata `regress, vce(cluster)`).22    case cluster(groups: [Int], smallSample: Bool)2324    public var label: String {25        switch self {26        case .nonrobust: "nonrobust"27        case .hc0: "HC0"28        case .hc1: "HC1"29        case .hc2: "HC2"30        case .hc3: "HC3"31        case .hac(let l, let s): "HAC (Newey–West, Bartlett, \(l) lags\(s ? ", small-sample" : ""))"32        case .cluster(let g, let s): "cluster (\(Set(g).count) groups\(s ? ", small-sample" : ""))"33        }34    }35    var isRobust: Bool { if case .nonrobust = self { false } else { true } }36}3738/// A fitted linear regression (OLS or WLS).39///40/// Conventions (identical to statsmodels `OLS`/`WLS`):41/// - `dfResidual` = n − k (k counts the intercept); `sigma2` = SSR / (n − k).42/// - R² is centred when the model has a constant, uncentred otherwise; adjusted R² = 1 − (n − c)/(n − k)·(1 − R²), c = 1 with a constant.43/// - `logLikelihood` is the Gaussian log-likelihood concentrated over σ² (σ̂² = SSR/n):44///   −n/2·(ln 2π + ln(SSR/n) + 1) (+ ½ Σ ln wᵢ for WLS); AIC = −2ℓ + 2k, BIC = −2ℓ + k ln n (σ² is not counted as a parameter).45/// - With the classical covariance, inference uses Student t(n − k) and the F test is the usual46///   (ESS/df_model)/(SSR/df_resid). With a robust covariance, p-values and intervals use the normal47///   distribution unless `useT` was requested (then t with n − k df, or G − 1 for clusters) and the48///   F statistic is the Wald test of all slopes = 0 divided by its df, referred to F(df_model, df_inference).49/// - All p-values of coefficients are two-sided; the F p-value is upper-tail.50public struct LinearModelResult: Codable, Sendable, Hashable {51    /// "OLS" or "WLS".52    public var method: String53    public var dependentName: String54    /// Coefficient names, intercept ("const") first when present.55    public var names: [String]56    /// True when `fit` added the constant column.57    public var hasIntercept: Bool58    /// True when the design contains a constant column (added by `fit` or supplied among the regressors);59    /// decides between centred and uncentred R².60    public var hasConstant: Bool61    public var nobs: Int62    /// Number of regressors excluding the constant.63    public var dfModel: Int64    public var dfResidual: Int6566    public var coefficients: [Double]67    public var standardErrors: [Double]68    /// t (or z) statistics.69    public var statistics: [Double]70    /// Two-sided p-values.71    public var pValues: [Double]72    public var ciLower: [Double]73    public var ciUpper: [Double]74    public var confidenceLevel: Double75    /// Covariance matrix of the coefficients.76    public var covariance: Matrix77    public var covarianceType: CovarianceType78    /// True when inference uses Student t, false for the normal distribution.79    public var usesT: Bool80    /// Degrees of freedom used for t and for the denominator of F.81    public var dfInference: Double8283    public var rSquared: Double84    public var adjustedRSquared: Double85    /// nil when the model has no regressor besides the constant.86    public var fStatistic: Double?87    /// Upper-tail p-value of `fStatistic`.88    public var fPValue: Double?89    /// SSR / (n − k).90    public var sigma2: Double91    public var residualStandardError: Double { sigma2.squareRoot() }92    /// Sum of squared (weighted, for WLS) residuals.93    public var ssr: Double94    /// Total sum of squares (centred when the model has a constant; weighted for WLS).95    public var tss: Double96    public var logLikelihood: Double97    public var aic: Double98    public var bic: Double99100    public var fitted: [Double]101    /// y − ŷ in the original units (unweighted, also for WLS).102    public var residuals: [Double]103    /// Diagonal of the hat matrix (of the weighted model for WLS).104    public var leverage: [Double]105    /// Internally studentised residuals eᵢ / (σ̂ √(1 − hᵢ)). An observation with leverage 1 is fitted106    /// exactly (its residual is 0 and the ratio is undefined): it is reported as 0, as is its Cook's distance.107    public var studentizedResiduals: [Double]108    /// Cook's distance Dᵢ = rᵢ² hᵢ / (k (1 − hᵢ)), rᵢ the internally studentised residual.109    public var cooksDistance: [Double]110    /// Variance inflation factors 1/(1 − R²ⱼ), regressing each column on all the others111    /// (constant included); nil for the constant itself and in models without a constant column.112    public var vif: [Double?]113    /// Ratio of the largest to the smallest singular value of the design matrix (unscaled columns, as statsmodels).114    public var conditionNumber: Double115116    /// Design matrix actually used (constant column first when `hasIntercept`).117    public var design: Matrix118    public var y: [Double]119    /// Observation weights (WLS), nil for OLS.120    public var weights: [Double]?121122    /// Predictions for new rows of regressors (without the constant column; it is added when the model has one).123    public func predict(_ x: Matrix) throws -> [Double] {124        let expect = names.count - (hasIntercept ? 1 : 0)125        guard x.cols == expect else { throw StatError.invalidInput("expected \(expect) regressors, got \(x.cols)") }126        return (0..<x.rows).map { r in127            var s = hasIntercept ? coefficients[0] : 0128            for c in 0..<x.cols { s += x[r, c] * coefficients[c + (hasIntercept ? 1 : 0)] }129            return s130        }131    }132133    /// Design and residuals of the transformed (√w-weighted) model on which the algebra is done.134    var whitened: (x: Matrix, e: [Double]) {135        guard let w = weights else { return (design, residuals) }136        var x = design137        var e = residuals138        for i in 0..<x.rows {139            let s = w[i].squareRoot()140            for j in 0..<x.cols { x[i, j] *= s }141            e[i] *= s142        }143        return (x, e)144    }145}146147public enum LinearRegression {148    /// Ordinary (or, with `weights`, weighted) least squares by Householder QR (LAPACK `dgeqrf`).149    ///150    /// - Parameters:151    ///   - y: dependent variable.152    ///   - x: regressors, one column per variable, WITHOUT a constant column.153    ///   - names: one name per column of `x` (default x1, x2, …).154    ///   - intercept: add a constant ("const") as the first column.155    ///   - weights: WLS weights wᵢ > 0, proportional to the inverse error variances (statsmodels `WLS`).156    ///   - covariance: covariance estimator, see `CovarianceType`.157    ///   - useT: nil = Student t for the classical covariance and normal for robust ones (statsmodels defaults).158    /// - Throws: `StatError.insufficientData` when n ≤ k, `StatError.singular` when the design is rank deficient159    ///   (perfect collinearity, a constant regressor next to the intercept, the dummy-variable trap).160    public static func fit(y: [Double], x: Matrix, names: [String]? = nil, intercept: Bool = true, weights: [Double]? = nil,161                           covariance: CovarianceType = .nonrobust, confidenceLevel: Double = 0.95, useT: Bool? = nil,162                           dependentName: String = "y") throws -> LinearModelResult {163        let n = y.count164        guard x.rows == n else { throw StatError.invalidInput("y has \(n) observations but X has \(x.rows) rows") }165        guard confidenceLevel > 0, confidenceLevel < 1 else { throw StatError.invalidInput("confidence level must be in (0, 1)") }166        if let names, names.count != x.cols { throw StatError.invalidInput("\(names.count) names for \(x.cols) regressors") }167        if let w = weights {168            guard w.count == n else { throw StatError.invalidInput("one weight per observation is required") }169            guard w.allSatisfy({ $0 > 0 && $0.isFinite }) else { throw StatError.invalidInput("weights must be positive and finite") }170        }171        var colNames = names ?? (0..<x.cols).map { "x\($0 + 1)" }172        var design = x173        if intercept {174            var d = Matrix(rows: n, cols: x.cols + 1)175            for i in 0..<n {176                d[i, 0] = 1177                for j in 0..<x.cols { d[i, j + 1] = x[i, j] }178            }179            design = d180            colNames.insert("const", at: 0)181        }182        let k = design.cols183        guard k > 0 else { throw StatError.invalidInput("the model has no regressors") }184        guard n > k else { throw StatError.insufficientData("\(n) observations for \(k) coefficients: at least \(k + 1) are needed") }185186        // transformed model187        var xw = design, yw = y188        if let w = weights {189            for i in 0..<n {190                let s = w[i].squareRoot()191                for j in 0..<k { xw[i, j] *= s }192                yw[i] *= s193            }194        }195        let core = try RegressionMath.solve(xw, yw, names: colNames)196        let hasConst = intercept || (0..<k).contains { RegressionMath.isConstantColumn(design, $0) }197        let dfResid = n - k198        let dfModel = k - (hasConst ? 1 : 0)199        let ssr = core.ssr200        let sigma2 = ssr / Double(dfResid)201        let tss: Double202        if hasConst {203            if let w = weights {204                let sw = Descriptive.sum(w)205                let mean = zip(w, y).reduce(0) { $0 + $1.0 * $1.1 } / sw206                tss = zip(w, y).reduce(0) { $0 + $1.0 * ($1.1 - mean) * ($1.1 - mean) }207            } else { tss = RegressionMath.centeredSS(y) }208        } else { tss = RegressionMath.uncenteredSS(yw) }209        guard tss > 0 else { throw StatError.invalidInput("the dependent variable is constant: there is nothing to explain") }210        let r2 = 1 - ssr / tss211        let adj = 1 - Double(n - (hasConst ? 1 : 0)) / Double(dfResid) * (1 - r2)212        let nd = Double(n)213        var llf = -nd / 2 * (log(2 * Double.pi) + log(ssr / nd) + 1)214        if let w = weights { llf += 0.5 * w.reduce(0) { $0 + log($1) } }215216        let fitted = design * core.beta217        let resid = zip(y, fitted).map { $0 - $1 }218        let lev = core.leverage219        var student = [Double](repeating: 0, count: n), cooks = [Double](repeating: 0, count: n)220        for i in 0..<n {221            let one = 1 - lev[i]222            guard one > 1e-12, sigma2 > 0 else { continue }223            let r = core.residuals[i] / (sigma2 * one).squareRoot()224            student[i] = r225            cooks[i] = r * r * lev[i] / (Double(k) * one)226        }227        // VIF228        var vif = [Double?](repeating: nil, count: k)229        if hasConst, k >= 2 {230            for j in 0..<k where !RegressionMath.isConstantColumn(design, j) {231                var others = Matrix(rows: n, cols: k - 1)232                var target = [Double](repeating: 0, count: n)233                for i in 0..<n {234                    var c = 0235                    for jj in 0..<k {236                        if jj == j { target[i] = design[i, jj] } else { others[i, c] = design[i, jj]; c += 1 }237                    }238                }239                if let aux = try? RegressionMath.solve(others, target) {240                    let t = RegressionMath.centeredSS(target)241                    vif[j] = aux.ssr > 0 ? t / aux.ssr : .infinity242                }243            }244        }245        let cond = (try? LinearAlgebra.conditionNumber(design)) ?? .nan246247        var result = LinearModelResult(248            method: weights == nil ? "OLS" : "WLS", dependentName: dependentName, names: colNames, hasIntercept: intercept, hasConstant: hasConst,249            nobs: n, dfModel: dfModel, dfResidual: dfResid, coefficients: core.beta, standardErrors: [], statistics: [], pValues: [],250            ciLower: [], ciUpper: [], confidenceLevel: confidenceLevel, covariance: Matrix(rows: k, cols: k), covarianceType: .nonrobust,251            usesT: true, dfInference: Double(dfResid), rSquared: r2, adjustedRSquared: adj, fStatistic: nil, fPValue: nil,252            sigma2: sigma2, ssr: ssr, tss: tss, logLikelihood: llf, aic: -2 * llf + 2 * Double(k), bic: -2 * llf + log(nd) * Double(k),253            fitted: fitted, residuals: resid, leverage: lev, studentizedResiduals: student, cooksDistance: cooks, vif: vif,254            conditionNumber: cond, design: design, y: y, weights: weights)255        try applyCovariance(&result, covariance, useT: useT, xtxInverse: core.xtxInverse)256        return result257    }258259    /// Convenience overload taking the regressors as columns.260    public static func fit(y: [Double], columns: [[Double]], names: [String]? = nil, intercept: Bool = true, weights: [Double]? = nil,261                           covariance: CovarianceType = .nonrobust, confidenceLevel: Double = 0.95, useT: Bool? = nil,262                           dependentName: String = "y") throws -> LinearModelResult {263        guard columns.allSatisfy({ $0.count == y.count }) else { throw StatError.invalidInput("every regressor needs one value per observation") }264        let x = columns.isEmpty ? Matrix(rows: y.count, cols: 0) : Matrix(columns: columns)265        return try fit(y: y, x: x, names: names, intercept: intercept, weights: weights, covariance: covariance,266                       confidenceLevel: confidenceLevel, useT: useT, dependentName: dependentName)267    }268269    /// Covariance matrix of the coefficients of a fitted model under another estimator.270    public static func covarianceMatrix(_ model: LinearModelResult, _ type: CovarianceType) throws -> Matrix {271        let (x, e) = model.whitened272        let n = x.rows, k = x.cols273        let yw = zip(model.y, model.weights ?? Array(repeating: 1, count: n)).map { $0 * $1.squareRoot() }274        let core = try RegressionMath.solve(x, yw, names: model.names)275        return try covarianceMatrix(type, x: x, e: e, leverage: model.leverage, sigma2: model.sigma2, xtxInverse: core.xtxInverse, n: n, k: k)276    }277278    static func covarianceMatrix(_ type: CovarianceType, x: Matrix, e: [Double], leverage h: [Double], sigma2: Double,279                                 xtxInverse bread: Matrix, n: Int, k: Int) throws -> Matrix {280        switch type {281        case .nonrobust:282            return sigma2 * bread283        case .hc0:284            return RegressionMath.sandwich(bread, RegressionMath.weightedGram(x, e.map { $0 * $0 }))285        case .hc1:286            return Double(n) / Double(n - k) * RegressionMath.sandwich(bread, RegressionMath.weightedGram(x, e.map { $0 * $0 }))287        case .hc2:288            return RegressionMath.sandwich(bread, RegressionMath.weightedGram(x, zip(e, h).map { $0 * $0 / (1 - $1) }))289        case .hc3:290            return RegressionMath.sandwich(bread, RegressionMath.weightedGram(x, zip(e, h).map { $0 * $0 / ((1 - $1) * (1 - $1)) }))291        case .hac(let maxLags, let small):292            guard maxLags >= 0, maxLags < n else { throw StatError.invalidInput("HAC lags must be between 0 and n − 1") }293            var s = RegressionMath.weightedGram(x, e.map { $0 * $0 })294            for lag in 0..<maxLags {295                let l = lag + 1296                let w = 1 - Double(l) / Double(maxLags + 1)297                var g = Matrix(rows: k, cols: k)298                for t in l..<n {299                    let et = e[t] * e[t - l]300                    for a in 0..<k { let xa = x[t, a] * et; for b in 0..<k { g[a, b] += xa * x[t - l, b] } }301                }302                s = s + w * (g + g.transposed)303            }304            let v = RegressionMath.sandwich(bread, s)305            return small ? Double(n) / Double(n - k) * v : v306        case .cluster(let groups, let small):307            guard groups.count == n else { throw StatError.invalidInput("one cluster label per observation is required") }308            var sums = [Int: [Double]]()309            for i in 0..<n {310                var acc = sums[groups[i]] ?? [Double](repeating: 0, count: k)311                for j in 0..<k { acc[j] += x[i, j] * e[i] }312                sums[groups[i]] = acc313            }314            let g = sums.count315            guard g >= 2 else { throw StatError.insufficientData("cluster-robust covariance needs at least 2 clusters") }316            var s = Matrix(rows: k, cols: k)317            for v in sums.values { for a in 0..<k { for b in 0..<k { s[a, b] += v[a] * v[b] } } }318            let cov = RegressionMath.sandwich(bread, s)319            guard small else { return cov }320            return Double(g) / Double(g - 1) * Double(n - 1) / Double(n - k) * cov321        }322    }323324    /// Returns the same fit with standard errors, tests and intervals recomputed under another covariance estimator.325    public static func reestimate(_ model: LinearModelResult, covariance: CovarianceType, useT: Bool? = nil) throws -> LinearModelResult {326        var out = model327        let (x, _) = model.whitened328        let yw = zip(model.y, model.weights ?? Array(repeating: 1, count: model.nobs)).map { $0 * $1.squareRoot() }329        let core = try RegressionMath.solve(x, yw, names: model.names)330        try applyCovariance(&out, covariance, useT: useT, xtxInverse: core.xtxInverse)331        return out332    }333334    private static func applyCovariance(_ r: inout LinearModelResult, _ type: CovarianceType, useT: Bool?, xtxInverse: Matrix) throws {335        let (x, e) = r.whitened336        let n = r.nobs, k = r.names.count337        let cov = try covarianceMatrix(type, x: x, e: e, leverage: r.leverage, sigma2: r.sigma2, xtxInverse: xtxInverse, n: n, k: k)338        let t = useT ?? !type.isRobust339        var df = Double(r.dfResidual)340        if case .cluster(let g, _) = type { df = Double(Set(g).count - 1) }341        r.covariance = cov342        r.covarianceType = type343        r.usesT = t344        r.dfInference = df345        let se = (0..<k).map { cov[$0, $0].squareRoot() }346        let stat = (0..<k).map { r.coefficients[$0] / se[$0] }347        let tDist = StudentTDistribution(df: df)348        let alpha = 1 - r.confidenceLevel349        let crit = t ? tDist.isf(alpha / 2) : NormalDistribution.standard.isf(alpha / 2)350        r.standardErrors = se351        r.statistics = stat352        r.pValues = stat.map { s in353            return t ? 2 * tDist.sf(abs(s)) : 2 * NormalDistribution.standard.sf(abs(s))354        }355        r.ciLower = (0..<k).map { r.coefficients[$0] - crit * se[$0] }356        r.ciUpper = (0..<k).map { r.coefficients[$0] + crit * se[$0] }357        // overall F358        r.fStatistic = nil; r.fPValue = nil359        var fStat: Double360        guard r.dfModel > 0 else { return }361        if case .nonrobust = type {362            let ess = r.tss - r.ssr363            fStat = (ess / Double(r.dfModel)) / r.sigma2364        } else {365            let slopes = r.hasConstant ? (0..<k).filter { !RegressionMath.isConstantColumn(r.design, $0) } : Array(0..<k)366            guard !slopes.isEmpty, let w = try? RegressionMath.wald(r.coefficients, cov, slopes) else { return }367            fStat = w / Double(slopes.count)368        }369        guard fStat.isFinite else { return }370        r.fStatistic = fStat371        r.fPValue = FisherFDistribution(d1: Double(r.dfModel), d2: df).sf(fStat)372    }373}374
Files main
Menu

Theme