Repository Public
1import Foundation23/// Covariance estimator for the fixed-effects panel estimator.4public enum PanelCovariance: Codable, Sendable, Hashable {5 /// Classical σ²(X̃ᵀX̃)⁻¹ with σ² = SSR / (N − k − absorbed effects).6 case unadjusted7 /// White heteroskedasticity-robust sandwich on the within-transformed data, scaled by N / dfResidual8 /// (the absorbed effects count as estimated parameters; linearmodels `cov_type='robust'`).9 case robust10 /// Clustered by entity (robust to arbitrary serial correlation within an entity).11 case clusteredByEntity(correction: PanelClusterCorrection)1213 public var label: String {14 switch self {15 case .unadjusted: "unadjusted"16 case .robust: "robust (White)"17 case .clusteredByEntity(let c):18 switch c {19 case .none: "clustered by entity (no small-sample correction)"20 case .linearmodels: "clustered by entity (linearmodels scaling)"21 case .lsdv: "clustered by entity (LSDV small-sample correction)"22 }23 }24 }25}2627/// Small-sample factor applied to the entity-clustered covariance.28public enum PanelClusterCorrection: String, Codable, Sendable, Hashable {29 /// The plain sandwich (X̃ᵀX̃)⁻¹ (Σ_g s_g s_gᵀ) (X̃ᵀX̃)⁻¹, no factor.30 case none31 /// What linearmodels `PanelOLS(...).fit(cov_type='clustered', cluster_entity=True)` reports: the plain32 /// sandwich × N/(N − k) with entity effects only (the effects nested in the clusters are not counted),33 /// and × N/dfResidual when time effects are also included.34 case linearmodels35 /// G/(G − 1) · (N − 1)/(N − K) with K counting the slopes AND every absorbed effect — what36 /// statsmodels reports for OLS on explicit entity dummies with `cov_type='cluster'`.37 case lsdv38}3940/// Fixed-effects (within) panel regression.41///42/// Conventions: `dfResidual` = N − k − (number of entities) − (T − 1 with time effects);43/// `rSquared` is 1 − SSR/TSS of the dependent variable after the included effects are removed44/// (linearmodels `rsquared`; equals the "within" R² for entity effects only). With the unadjusted45/// covariance coefficient p-values use Student t(dfResidual); with robust/clustered covariances they use46/// the normal distribution unless `useT` (then t with dfResidual, or G − 1 when clustered). All47/// coefficient p-values are two-sided; F and χ² p-values are upper-tail.48public struct PanelResult: Codable, Sendable, Hashable {49 public var dependentName: String50 public var names: [String]51 public var coefficients: [Double]52 public var standardErrors: [Double]53 public var statistics: [Double]54 public var pValues: [Double]55 public var ciLower: [Double]56 public var ciUpper: [Double]57 public var confidenceLevel: Double58 public var covariance: Matrix59 public var covarianceType: PanelCovariance60 public var usesT: Bool61 public var dfInference: Double6263 public var nobs: Int64 public var entityCount: Int65 /// Number of distinct time periods (0 when no time index was given).66 public var timeCount: Int67 public var timeEffects: Bool68 public var dfModel: Int69 public var dfResidual: Int70 public var ssr: Double71 public var sigma2: Double72 public var rSquared: Double73 /// Classical F test that all slopes are zero, F(k, dfResidual).74 public var fStatistic: Double75 public var fPValue: Double76 /// Robust test that all slopes are zero: Wald statistic / k, referred to F(k, dfResidual)77 /// (linearmodels `f_statistic_robust`); nil with the unadjusted covariance.78 public var robustFStatistic: Double?79 public var robustFPValue: Double?80 /// F test that all the fixed effects are equal (pooled OLS adequate), F(dfPoolability, dfResidual).81 public var poolabilityF: Double82 public var poolabilityPValue: Double83 public var dfPoolability: Int8485 /// Entity labels in order of first appearance and their estimated effects αᵢ.86 public var entityLabels: [String]87 public var entityEffects: [Double]88 /// Time labels (sorted) and effects γₜ, the first period being the reference (0). Empty without time effects.89 public var timeLabels: [String]90 public var timeEffectValues: [Double]91 /// y − x'β − αᵢ − γₜ.92 public var residuals: [Double]93 /// αᵢ + γₜ for every observation.94 public var estimatedEffects: [Double]95}9697public enum PanelRegression {98 /// Fixed-effects (within) estimator: entity means are removed from every variable; time effects,99 /// when requested, enter as period dummies (first period dropped) that are demeaned alongside,100 /// which is exact for unbalanced panels too.101 ///102 /// - Parameters:103 /// - x: regressors without a constant; time-invariant regressors are not identified and throw `.singular`.104 /// - entity: entity identifier of each observation.105 /// - time: period identifier of each observation (required for `timeEffects`).106 public static func fixedEffects<E: Hashable, T: Hashable & Comparable>(107 y: [Double], x: Matrix, names: [String]? = nil, entity: [E], time: [T]? = nil, timeEffects: Bool = false,108 covariance: PanelCovariance = .unadjusted, confidenceLevel: Double = 0.95, useT: Bool? = nil, dependentName: String = "y"109 ) throws -> PanelResult {110 let n = y.count, k = x.cols111 guard x.rows == n, entity.count == n else { throw StatError.invalidInput("y, X and the entity index must have the same length") }112 guard k > 0 else { throw StatError.invalidInput("the model has no regressors") }113 if let names, names.count != k { throw StatError.invalidInput("\(names.count) names for \(k) regressors") }114 if let time, time.count != n { throw StatError.invalidInput("the time index must have one value per observation") }115 if timeEffects && time == nil { throw StatError.invalidInput("time effects need a time index") }116 guard confidenceLevel > 0, confidenceLevel < 1 else { throw StatError.invalidInput("confidence level must be in (0, 1)") }117 let colNames = names ?? (0..<k).map { "x\($0 + 1)" }118119 var entityIndex = [E: Int](), entityOrder = [E](), eIdx = [Int](repeating: 0, count: n)120 for i in 0..<n {121 if let j = entityIndex[entity[i]] { eIdx[i] = j } else {122 entityIndex[entity[i]] = entityOrder.count; eIdx[i] = entityOrder.count; entityOrder.append(entity[i])123 }124 }125 let g = entityOrder.count126 guard g >= 2 else { throw StatError.insufficientData("a panel needs at least 2 entities") }127 var timeLevels = [T](), tIdx = [Int](repeating: 0, count: n)128 if let time {129 timeLevels = Array(Set(time)).sorted()130 var map = [T: Int]()131 for (i, t) in timeLevels.enumerated() { map[t] = i }132 for i in 0..<n { tIdx[i] = map[time[i]]! }133 }134 let extra = timeEffects ? timeLevels.count - 1 : 0135 let kTotal = k + extra136 let dfResid = n - kTotal - g137 guard dfResid > 0 else { throw StatError.insufficientData("not enough observations for \(kTotal) coefficients and \(g) entity effects") }138139 var counts = [Double](repeating: 0, count: g)140 for i in 0..<n { counts[eIdx[i]] += 1 }141 func demean(_ v: [Double]) -> (demeaned: [Double], means: [Double]) {142 var sums = [Double](repeating: 0, count: g)143 for i in 0..<n { sums[eIdx[i]] += v[i] }144 let means = (0..<g).map { sums[$0] / counts[$0] }145 return ((0..<n).map { v[$0] - means[eIdx[$0]] }, means)146 }147 // raw columns: slopes then time dummies148 var raw = x.columnArrays149 if timeEffects {150 for t in 1..<timeLevels.count { raw.append((0..<n).map { tIdx[$0] == t ? 1.0 : 0.0 }) }151 }152 let (yd, yMeans) = demean(y)153 var colMeans = [[Double]]()154 var demeanedCols = [[Double]]()155 for c in raw { let d = demean(c); demeanedCols.append(d.demeaned); colMeans.append(d.means) }156 let xd = Matrix(columns: demeanedCols)157 let allNames = colNames + (timeEffects ? timeLevels.dropFirst().map { "time[\($0)]" } : [])158 let core: RegressionMath.Core159 do { core = try RegressionMath.solve(xd, yd, names: allNames) } catch StatError.singular(let m) {160 throw StatError.singular(m + " after removing the entity means (time-invariant regressors are absorbed by the fixed effects)")161 }162 let ssr = core.ssr163 let sigma2 = ssr / Double(dfResid)164165 // covariance of all coefficients (slopes + time dummies)166 let covAll: Matrix167 var isRobust = true168 var dfInf = Double(dfResid)169 switch covariance {170 case .unadjusted:171 covAll = sigma2 * core.xtxInverse; isRobust = false172 case .robust:173 covAll = Double(n) / Double(dfResid) * RegressionMath.sandwich(core.xtxInverse, RegressionMath.weightedGram(xd, core.residuals.map { $0 * $0 }))174 case .clusteredByEntity(let correction):175 var c = try LinearRegression.covarianceMatrix(.cluster(groups: eIdx, smallSample: false), x: xd, e: core.residuals, leverage: [],176 sigma2: sigma2, xtxInverse: core.xtxInverse, n: n, k: kTotal)177 switch correction {178 case .none: break179 case .linearmodels: c = Double(n) / Double(timeEffects ? dfResid : n - k) * c180 case .lsdv: c = Double(g) / Double(g - 1) * Double(n - 1) / Double(n - kTotal - g) * c181 }182 covAll = c183 dfInf = Double(g - 1)184 }185 let t = useT ?? !isRobust186 if !t { dfInf = Double(dfResid) }187 var cov = Matrix(rows: k, cols: k)188 for a in 0..<k { for b in 0..<k { cov[a, b] = covAll[a, b] } }189 let beta = Array(core.beta[0..<k])190 let se = (0..<k).map { cov[$0, $0].squareRoot() }191 let stat = (0..<k).map { beta[$0] / se[$0] }192 let tDist = StudentTDistribution(df: dfInf)193 let crit = t ? tDist.isf((1 - confidenceLevel) / 2) : NormalDistribution.standard.isf((1 - confidenceLevel) / 2)194 let pvals = stat.map { t ? 2 * tDist.sf(abs($0)) : 2 * NormalDistribution.standard.sf(abs($0)) }195196 // TSS with the effects removed: for two-way models, residual SS of ỹ on the demeaned time dummies197 var tssEff = RegressionMath.uncenteredSS(yd)198 if timeEffects, extra > 0 {199 tssEff = try RegressionMath.solve(Matrix(columns: Array(demeanedCols[k...])), yd).ssr200 }201 let f = ((tssEff - ssr) / Double(k)) / sigma2202 var wald: Double?, waldP: Double?203 if isRobust, let w = try? RegressionMath.wald(core.beta, covAll, Array(0..<k)) {204 wald = w / Double(k); waldP = FisherFDistribution(d1: Double(k), d2: Double(dfResid)).sf(w / Double(k))205 }206 // poolability: pooled OLS with a constant on the slopes only207 var pooledDesign = Matrix(rows: n, cols: k + 1)208 for i in 0..<n { pooledDesign[i, 0] = 1; for j in 0..<k { pooledDesign[i, j + 1] = x[i, j] } }209 let dfPool = g - 1 + extra210 var poolF = Double.nan211 if let pooled = try? RegressionMath.solve(pooledDesign, y) {212 poolF = ((pooled.ssr - ssr) / Double(dfPool)) / sigma2213 }214 // effects215 var alpha = [Double](repeating: 0, count: g)216 for e in 0..<g {217 var a = yMeans[e]218 for j in 0..<kTotal { a -= colMeans[j][e] * core.beta[j] }219 alpha[e] = a220 }221 let gamma: [Double] = timeEffects ? [0] + Array(core.beta[k...]) : []222 let effects = (0..<n).map { alpha[eIdx[$0]] + (timeEffects ? gamma[tIdx[$0]] : 0) }223224 return PanelResult(225 dependentName: dependentName, names: colNames, coefficients: beta, standardErrors: se, statistics: stat, pValues: pvals,226 ciLower: (0..<k).map { beta[$0] - crit * se[$0] }, ciUpper: (0..<k).map { beta[$0] + crit * se[$0] },227 confidenceLevel: confidenceLevel, covariance: cov, covarianceType: covariance, usesT: t, dfInference: dfInf,228 nobs: n, entityCount: g, timeCount: timeLevels.count, timeEffects: timeEffects, dfModel: k, dfResidual: dfResid,229 ssr: ssr, sigma2: sigma2, rSquared: 1 - ssr / tssEff, fStatistic: f,230 fPValue: FisherFDistribution(d1: Double(k), d2: Double(dfResid)).sf(f), robustFStatistic: wald, robustFPValue: waldP,231 poolabilityF: poolF, poolabilityPValue: FisherFDistribution(d1: Double(dfPool), d2: Double(dfResid)).sf(poolF), dfPoolability: dfPool,232 entityLabels: entityOrder.map { String(describing: $0) }, entityEffects: alpha,233 timeLabels: timeEffects ? timeLevels.map { String(describing: $0) } : [], timeEffectValues: gamma,234 residuals: core.residuals, estimatedEffects: effects)235 }236237 /// Entity fixed effects without a time index.238 public static func fixedEffects<E: Hashable>(y: [Double], x: Matrix, names: [String]? = nil, entity: [E],239 covariance: PanelCovariance = .unadjusted, confidenceLevel: Double = 0.95,240 useT: Bool? = nil, dependentName: String = "y") throws -> PanelResult {241 try fixedEffects(y: y, x: x, names: names, entity: entity, time: Optional<[Int]>.none, timeEffects: false,242 covariance: covariance, confidenceLevel: confidenceLevel, useT: useT, dependentName: dependentName)243 }244}245