SPB Git

Repository Public

Panel.swift 13.3 KB 245 lines swift

1import Foundation23/// Covariance estimator for the fixed-effects panel estimator.4public enum PanelCovariance: Codable, Sendable, Hashable {5    /// Classical σ²(X̃ᵀX̃)⁻¹ with σ² = SSR / (N − k − absorbed effects).6    case unadjusted7    /// White heteroskedasticity-robust sandwich on the within-transformed data, scaled by N / dfResidual8    /// (the absorbed effects count as estimated parameters; linearmodels `cov_type='robust'`).9    case robust10    /// Clustered by entity (robust to arbitrary serial correlation within an entity).11    case clusteredByEntity(correction: PanelClusterCorrection)1213    public var label: String {14        switch self {15        case .unadjusted: "unadjusted"16        case .robust: "robust (White)"17        case .clusteredByEntity(let c):18            switch c {19            case .none: "clustered by entity (no small-sample correction)"20            case .linearmodels: "clustered by entity (linearmodels scaling)"21            case .lsdv: "clustered by entity (LSDV small-sample correction)"22            }23        }24    }25}2627/// Small-sample factor applied to the entity-clustered covariance.28public enum PanelClusterCorrection: String, Codable, Sendable, Hashable {29    /// The plain sandwich (X̃ᵀX̃)⁻¹ (Σ_g s_g s_gᵀ) (X̃ᵀX̃)⁻¹, no factor.30    case none31    /// What linearmodels `PanelOLS(...).fit(cov_type='clustered', cluster_entity=True)` reports: the plain32    /// sandwich × N/(N − k) with entity effects only (the effects nested in the clusters are not counted),33    /// and × N/dfResidual when time effects are also included.34    case linearmodels35    /// G/(G − 1) · (N − 1)/(N − K) with K counting the slopes AND every absorbed effect — what36    /// statsmodels reports for OLS on explicit entity dummies with `cov_type='cluster'`.37    case lsdv38}3940/// Fixed-effects (within) panel regression.41///42/// Conventions: `dfResidual` = N − k − (number of entities) − (T − 1 with time effects);43/// `rSquared` is 1 − SSR/TSS of the dependent variable after the included effects are removed44/// (linearmodels `rsquared`; equals the "within" R² for entity effects only). With the unadjusted45/// covariance coefficient p-values use Student t(dfResidual); with robust/clustered covariances they use46/// the normal distribution unless `useT` (then t with dfResidual, or G − 1 when clustered). All47/// coefficient p-values are two-sided; F and χ² p-values are upper-tail.48public struct PanelResult: Codable, Sendable, Hashable {49    public var dependentName: String50    public var names: [String]51    public var coefficients: [Double]52    public var standardErrors: [Double]53    public var statistics: [Double]54    public var pValues: [Double]55    public var ciLower: [Double]56    public var ciUpper: [Double]57    public var confidenceLevel: Double58    public var covariance: Matrix59    public var covarianceType: PanelCovariance60    public var usesT: Bool61    public var dfInference: Double6263    public var nobs: Int64    public var entityCount: Int65    /// Number of distinct time periods (0 when no time index was given).66    public var timeCount: Int67    public var timeEffects: Bool68    public var dfModel: Int69    public var dfResidual: Int70    public var ssr: Double71    public var sigma2: Double72    public var rSquared: Double73    /// Classical F test that all slopes are zero, F(k, dfResidual).74    public var fStatistic: Double75    public var fPValue: Double76    /// Robust test that all slopes are zero: Wald statistic / k, referred to F(k, dfResidual)77    /// (linearmodels `f_statistic_robust`); nil with the unadjusted covariance.78    public var robustFStatistic: Double?79    public var robustFPValue: Double?80    /// F test that all the fixed effects are equal (pooled OLS adequate), F(dfPoolability, dfResidual).81    public var poolabilityF: Double82    public var poolabilityPValue: Double83    public var dfPoolability: Int8485    /// Entity labels in order of first appearance and their estimated effects αᵢ.86    public var entityLabels: [String]87    public var entityEffects: [Double]88    /// Time labels (sorted) and effects γₜ, the first period being the reference (0). Empty without time effects.89    public var timeLabels: [String]90    public var timeEffectValues: [Double]91    /// y − x'β − αᵢ − γₜ.92    public var residuals: [Double]93    /// αᵢ + γₜ for every observation.94    public var estimatedEffects: [Double]95}9697public enum PanelRegression {98    /// Fixed-effects (within) estimator: entity means are removed from every variable; time effects,99    /// when requested, enter as period dummies (first period dropped) that are demeaned alongside,100    /// which is exact for unbalanced panels too.101    ///102    /// - Parameters:103    ///   - x: regressors without a constant; time-invariant regressors are not identified and throw `.singular`.104    ///   - entity: entity identifier of each observation.105    ///   - time: period identifier of each observation (required for `timeEffects`).106    public static func fixedEffects<E: Hashable, T: Hashable & Comparable>(107        y: [Double], x: Matrix, names: [String]? = nil, entity: [E], time: [T]? = nil, timeEffects: Bool = false,108        covariance: PanelCovariance = .unadjusted, confidenceLevel: Double = 0.95, useT: Bool? = nil, dependentName: String = "y"109    ) throws -> PanelResult {110        let n = y.count, k = x.cols111        guard x.rows == n, entity.count == n else { throw StatError.invalidInput("y, X and the entity index must have the same length") }112        guard k > 0 else { throw StatError.invalidInput("the model has no regressors") }113        if let names, names.count != k { throw StatError.invalidInput("\(names.count) names for \(k) regressors") }114        if let time, time.count != n { throw StatError.invalidInput("the time index must have one value per observation") }115        if timeEffects && time == nil { throw StatError.invalidInput("time effects need a time index") }116        guard confidenceLevel > 0, confidenceLevel < 1 else { throw StatError.invalidInput("confidence level must be in (0, 1)") }117        let colNames = names ?? (0..<k).map { "x\($0 + 1)" }118119        var entityIndex = [E: Int](), entityOrder = [E](), eIdx = [Int](repeating: 0, count: n)120        for i in 0..<n {121            if let j = entityIndex[entity[i]] { eIdx[i] = j } else {122                entityIndex[entity[i]] = entityOrder.count; eIdx[i] = entityOrder.count; entityOrder.append(entity[i])123            }124        }125        let g = entityOrder.count126        guard g >= 2 else { throw StatError.insufficientData("a panel needs at least 2 entities") }127        var timeLevels = [T](), tIdx = [Int](repeating: 0, count: n)128        if let time {129            timeLevels = Array(Set(time)).sorted()130            var map = [T: Int]()131            for (i, t) in timeLevels.enumerated() { map[t] = i }132            for i in 0..<n { tIdx[i] = map[time[i]]! }133        }134        let extra = timeEffects ? timeLevels.count - 1 : 0135        let kTotal = k + extra136        let dfResid = n - kTotal - g137        guard dfResid > 0 else { throw StatError.insufficientData("not enough observations for \(kTotal) coefficients and \(g) entity effects") }138139        var counts = [Double](repeating: 0, count: g)140        for i in 0..<n { counts[eIdx[i]] += 1 }141        func demean(_ v: [Double]) -> (demeaned: [Double], means: [Double]) {142            var sums = [Double](repeating: 0, count: g)143            for i in 0..<n { sums[eIdx[i]] += v[i] }144            let means = (0..<g).map { sums[$0] / counts[$0] }145            return ((0..<n).map { v[$0] - means[eIdx[$0]] }, means)146        }147        // raw columns: slopes then time dummies148        var raw = x.columnArrays149        if timeEffects {150            for t in 1..<timeLevels.count { raw.append((0..<n).map { tIdx[$0] == t ? 1.0 : 0.0 }) }151        }152        let (yd, yMeans) = demean(y)153        var colMeans = [[Double]]()154        var demeanedCols = [[Double]]()155        for c in raw { let d = demean(c); demeanedCols.append(d.demeaned); colMeans.append(d.means) }156        let xd = Matrix(columns: demeanedCols)157        let allNames = colNames + (timeEffects ? timeLevels.dropFirst().map { "time[\($0)]" } : [])158        let core: RegressionMath.Core159        do { core = try RegressionMath.solve(xd, yd, names: allNames) } catch StatError.singular(let m) {160            throw StatError.singular(m + " after removing the entity means (time-invariant regressors are absorbed by the fixed effects)")161        }162        let ssr = core.ssr163        let sigma2 = ssr / Double(dfResid)164165        // covariance of all coefficients (slopes + time dummies)166        let covAll: Matrix167        var isRobust = true168        var dfInf = Double(dfResid)169        switch covariance {170        case .unadjusted:171            covAll = sigma2 * core.xtxInverse; isRobust = false172        case .robust:173            covAll = Double(n) / Double(dfResid) * RegressionMath.sandwich(core.xtxInverse, RegressionMath.weightedGram(xd, core.residuals.map { $0 * $0 }))174        case .clusteredByEntity(let correction):175            var c = try LinearRegression.covarianceMatrix(.cluster(groups: eIdx, smallSample: false), x: xd, e: core.residuals, leverage: [],176                                                          sigma2: sigma2, xtxInverse: core.xtxInverse, n: n, k: kTotal)177            switch correction {178            case .none: break179            case .linearmodels: c = Double(n) / Double(timeEffects ? dfResid : n - k) * c180            case .lsdv: c = Double(g) / Double(g - 1) * Double(n - 1) / Double(n - kTotal - g) * c181            }182            covAll = c183            dfInf = Double(g - 1)184        }185        let t = useT ?? !isRobust186        if !t { dfInf = Double(dfResid) }187        var cov = Matrix(rows: k, cols: k)188        for a in 0..<k { for b in 0..<k { cov[a, b] = covAll[a, b] } }189        let beta = Array(core.beta[0..<k])190        let se = (0..<k).map { cov[$0, $0].squareRoot() }191        let stat = (0..<k).map { beta[$0] / se[$0] }192        let tDist = StudentTDistribution(df: dfInf)193        let crit = t ? tDist.isf((1 - confidenceLevel) / 2) : NormalDistribution.standard.isf((1 - confidenceLevel) / 2)194        let pvals = stat.map { t ? 2 * tDist.sf(abs($0)) : 2 * NormalDistribution.standard.sf(abs($0)) }195196        // TSS with the effects removed: for two-way models, residual SS of ỹ on the demeaned time dummies197        var tssEff = RegressionMath.uncenteredSS(yd)198        if timeEffects, extra > 0 {199            tssEff = try RegressionMath.solve(Matrix(columns: Array(demeanedCols[k...])), yd).ssr200        }201        let f = ((tssEff - ssr) / Double(k)) / sigma2202        var wald: Double?, waldP: Double?203        if isRobust, let w = try? RegressionMath.wald(core.beta, covAll, Array(0..<k)) {204            wald = w / Double(k); waldP = FisherFDistribution(d1: Double(k), d2: Double(dfResid)).sf(w / Double(k))205        }206        // poolability: pooled OLS with a constant on the slopes only207        var pooledDesign = Matrix(rows: n, cols: k + 1)208        for i in 0..<n { pooledDesign[i, 0] = 1; for j in 0..<k { pooledDesign[i, j + 1] = x[i, j] } }209        let dfPool = g - 1 + extra210        var poolF = Double.nan211        if let pooled = try? RegressionMath.solve(pooledDesign, y) {212            poolF = ((pooled.ssr - ssr) / Double(dfPool)) / sigma2213        }214        // effects215        var alpha = [Double](repeating: 0, count: g)216        for e in 0..<g {217            var a = yMeans[e]218            for j in 0..<kTotal { a -= colMeans[j][e] * core.beta[j] }219            alpha[e] = a220        }221        let gamma: [Double] = timeEffects ? [0] + Array(core.beta[k...]) : []222        let effects = (0..<n).map { alpha[eIdx[$0]] + (timeEffects ? gamma[tIdx[$0]] : 0) }223224        return PanelResult(225            dependentName: dependentName, names: colNames, coefficients: beta, standardErrors: se, statistics: stat, pValues: pvals,226            ciLower: (0..<k).map { beta[$0] - crit * se[$0] }, ciUpper: (0..<k).map { beta[$0] + crit * se[$0] },227            confidenceLevel: confidenceLevel, covariance: cov, covarianceType: covariance, usesT: t, dfInference: dfInf,228            nobs: n, entityCount: g, timeCount: timeLevels.count, timeEffects: timeEffects, dfModel: k, dfResidual: dfResid,229            ssr: ssr, sigma2: sigma2, rSquared: 1 - ssr / tssEff, fStatistic: f,230            fPValue: FisherFDistribution(d1: Double(k), d2: Double(dfResid)).sf(f), robustFStatistic: wald, robustFPValue: waldP,231            poolabilityF: poolF, poolabilityPValue: FisherFDistribution(d1: Double(dfPool), d2: Double(dfResid)).sf(poolF), dfPoolability: dfPool,232            entityLabels: entityOrder.map { String(describing: $0) }, entityEffects: alpha,233            timeLabels: timeEffects ? timeLevels.map { String(describing: $0) } : [], timeEffectValues: gamma,234            residuals: core.residuals, estimatedEffects: effects)235    }236237    /// Entity fixed effects without a time index.238    public static func fixedEffects<E: Hashable>(y: [Double], x: Matrix, names: [String]? = nil, entity: [E],239                                                 covariance: PanelCovariance = .unadjusted, confidenceLevel: Double = 0.95,240                                                 useT: Bool? = nil, dependentName: String = "y") throws -> PanelResult {241        try fixedEffects(y: y, x: x, names: names, entity: entity, time: Optional<[Int]>.none, timeEffects: false,242                         covariance: covariance, confidenceLevel: confidenceLevel, useT: useT, dependentName: dependentName)243    }244}245
Files main
Menu

Theme