SPB Git

Repository Public

Diagnostics.swift 15.4 KB 263 lines swift

1import Foundation2import SheetCore34/// Outcome of a specification / residual diagnostic test.5public struct RegressionDiagnostic: Codable, Sendable, Hashable, TableReport {6    public var name: String7    public var statistic: Double8    /// Reference distribution, e.g. "chi2(3)" or "F(2, 45)"; empty when no p-value is defined (Durbin–Watson).9    public var distribution: String10    /// Degrees of freedom of the reference distribution (one entry for χ², two for F).11    public var df: [Double]12    /// p-value; nil when the test has none (Durbin–Watson).13    public var pValue: Double?14    /// Which tail the p-value is: always "upper-tail (one-sided)" for the χ² and F tests here.15    public var tail: String16    public var nullHypothesis: String17    public var assumptions: String18    /// The F form of an LM test (or the χ² form of an F test), when the test has both.19    public var alternativeStatistic: Double?20    public var alternativeDistribution: String?21    public var alternativePValue: Double?2223    public func table() -> [[Value]] {24        var rows: [[Value]] = [25            [.text(name), .blank],26            Report.row("Statistic", statistic),27        ]28        if !distribution.isEmpty { rows.append(Report.row("Distribution", distribution)) }29        if let pValue { rows.append(Report.row("p-value", pValue)); rows.append(Report.row("Tail", tail)) }30        if let a = alternativeStatistic, let d = alternativeDistribution, let p = alternativePValue {31            rows.append(Report.row("Alternative form", d)); rows.append(Report.row("Statistic", a)); rows.append(Report.row("p-value", p))32        }33        rows.append(Report.row("H0", nullHypothesis))34        rows.append(Report.row("Assumptions", assumptions))35        return rows36    }37}3839/// Heteroskedasticity, autocorrelation, normality and functional-form diagnostics.40/// All χ² and F p-values are upper-tail.41public enum RegressionDiagnostics {42    private static let upper = "upper-tail (one-sided)"4344    private static func fmt(_ d: Double) -> String { d == d.rounded() ? String(Int(d)) : String(format: "%g", d) }4546    private static func chi2(_ name: String, _ stat: Double, _ df: Int, h0: String, assumptions: String) -> RegressionDiagnostic {47        RegressionDiagnostic(name: name, statistic: stat, distribution: "chi2(\(df))", df: [Double(df)],48                             pValue: ChiSquaredDistribution(df: Double(df)).sf(stat), tail: upper, nullHypothesis: h0, assumptions: assumptions)49    }5051    private static func withF(_ d: RegressionDiagnostic, _ f: Double, _ d1: Int, _ d2: Int) -> RegressionDiagnostic {52        var out = d53        out.alternativeStatistic = f54        out.alternativeDistribution = "F(\(d1), \(d2))"55        out.alternativePValue = FisherFDistribution(d1: Double(d1), d2: Double(d2)).sf(f)56        return out57    }5859    /// Auxiliary OLS of `target` on `x`: returns n·R² (centred when x has a constant), the F statistic of all non-constant60    /// regressors and its degrees of freedom.61    private static func auxiliary(_ target: [Double], _ x: Matrix, hasConstant: Bool) throws -> (lm: Double, f: Double, d1: Int, d2: Int, ssr: Double) {62        let n = x.rows, k = x.cols63        let core = try RegressionMath.solve(x, target)64        let tss = hasConstant ? RegressionMath.centeredSS(target) : RegressionMath.uncenteredSS(target)65        let r2 = 1 - core.ssr / tss66        let d1 = k - (hasConstant ? 1 : 0), d2 = n - k67        let f = ((tss - core.ssr) / Double(d1)) / (core.ssr / Double(d2))68        return (Double(n) * r2, f, d1, d2, core.ssr)69    }7071    // MARK: Heteroskedasticity7273    /// Breusch–Pagan test: regress the squared residuals on the model's regressors.74    /// - Parameter studentized: true (default) gives Koenker's robust version LM = n·R² (statsmodels75    ///   `het_breuschpagan(robust=True)`); false gives the original Breusch–Pagan statistic ESS/2 of76    ///   e²/σ̂² (σ̂² = SSR/n), which assumes normal errors.77    /// The F form is the overall F of the auxiliary regression. Requires a model with a constant.78    public static func breuschPagan(_ model: LinearModelResult, studentized: Bool = true) throws -> RegressionDiagnostic {79        guard model.hasConstant, model.names.count >= 2 else { throw StatError.invalidInput("Breusch–Pagan needs a constant and at least one regressor") }80        let (x, e) = model.whitened81        let n = Double(model.nobs)82        var e2 = e.map { $0 * $0 }83        let h0 = "homoskedasticity: the error variance does not depend on the regressors"84        if studentized {85            let a = try auxiliary(e2, x, hasConstant: true)86            let t = chi2("Breusch–Pagan (Koenker studentized)", a.lm, a.d1, h0: h0,87                         assumptions: "Linear model correctly specified; variance a function of the regressors under H1; no normality needed (Koenker).")88            return withF(t, a.f, a.d1, a.d2)89        }90        let s2 = Descriptive.sum(e2) / n91        e2 = e2.map { $0 / s2 }92        let core = try RegressionMath.solve(x, e2)93        let ess = RegressionMath.centeredSS(e2) - core.ssr94        let a = try auxiliary(e2, x, hasConstant: true)95        let t = chi2("Breusch–Pagan (original)", ess / 2, a.d1, h0: h0,96                     assumptions: "Linear model correctly specified; normally distributed errors (use the studentized version otherwise).")97        return withF(t, a.f, a.d1, a.d2)98    }99100    /// White's general test: squared residuals regressed on the regressors, their squares and all101    /// cross products (redundant columns removed). LM = n·R² ~ χ²(p − 1), p = number of auxiliary columns.102    /// Matches statsmodels `het_white`. Requires a model with a constant.103    public static func white(_ model: LinearModelResult) throws -> RegressionDiagnostic {104        guard model.hasConstant, model.names.count >= 2 else { throw StatError.invalidInput("White's test needs a constant and at least one regressor") }105        let (x, e) = model.whitened106        let cols = x.columnArrays107        var products = [[Double]]()108        // constant first so that it is the column kept among duplicates109        if let c = (0..<x.cols).first(where: { RegressionMath.isConstantColumn(model.design, $0) }) { products.append(cols[c].map { _ in 1.0 }) }110        for i in 0..<cols.count { for j in i..<cols.count { products.append(zip(cols[i], cols[j]).map(*)) } }111        let keep = RegressionMath.independentColumns(products)112        guard keep.count >= 2, keep.count < model.nobs else { throw StatError.insufficientData("not enough observations for White's auxiliary regression") }113        let aux = Matrix(columns: keep.map { products[$0] })114        let a = try auxiliary(e.map { $0 * $0 }, aux, hasConstant: true)115        let t = chi2("White", a.lm, a.d1, h0: "homoskedasticity (and correct functional form)",116                     assumptions: "No specific form of heteroskedasticity assumed; large-sample test; also reacts to misspecification.")117        return withF(t, a.f, a.d1, a.d2)118    }119120    // MARK: Autocorrelation121122    /// Durbin–Watson statistic Σ(eₜ − eₜ₋₁)² / Σeₜ² (≈ 2(1 − ρ̂₁); 2 = no first-order autocorrelation).123    public static func durbinWatson(_ residuals: [Double]) -> Double {124        guard residuals.count > 1 else { return .nan }125        var num = 0.0, den = residuals[0] * residuals[0]126        for t in 1..<residuals.count {127            let d = residuals[t] - residuals[t - 1]128            num += d * d129            den += residuals[t] * residuals[t]130        }131        return num / den132    }133134    /// Durbin–Watson as a diagnostic record (no p-value: its distribution depends on the regressors; use bounds tables or Breusch–Godfrey).135    public static func durbinWatson(_ model: LinearModelResult) -> RegressionDiagnostic {136        RegressionDiagnostic(name: "Durbin–Watson", statistic: durbinWatson(model.whitened.e), distribution: "", df: [], pValue: nil, tail: "",137                             nullHypothesis: "no first-order autocorrelation of the errors (statistic near 2)",138                             assumptions: "Observations in time order; regressors strictly exogenous (no lagged dependent variable); model has a constant.")139    }140141    /// Breusch–Godfrey LM test for serial correlation up to order `lags`: residuals regressed on the142    /// regressors and their own lags (pre-sample lags set to 0); LM = n·R² ~ χ²(lags); the F form143    /// tests the lag coefficients jointly, F(lags, n − k − lags). Matches statsmodels `acorr_breusch_godfrey`.144    public static func breuschGodfrey(_ model: LinearModelResult, lags: Int) throws -> RegressionDiagnostic {145        let n = model.nobs, k = model.names.count146        guard lags >= 1, n > k + lags else { throw StatError.invalidInput("Breusch–Godfrey needs 1 ≤ lags < n − k") }147        let (x, e) = model.whitened148        var aux = Matrix(rows: n, cols: k + lags)149        for t in 0..<n {150            for l in 1...lags where t - l >= 0 { aux[t, l - 1] = e[t - l] }151            for j in 0..<k { aux[t, lags + j] = x[t, j] }152        }153        let core = try RegressionMath.solve(aux, e)154        let tss = model.hasConstant ? RegressionMath.centeredSS(e) : RegressionMath.uncenteredSS(e)155        let lm = Double(n) * (1 - core.ssr / tss)156        // restricted model: residuals on X alone157        let restricted = try RegressionMath.solve(x, e).ssr158        let d2 = n - k - lags159        let f = ((restricted - core.ssr) / Double(lags)) / (core.ssr / Double(d2))160        let t = chi2("Breusch–Godfrey (\(lags) lag\(lags == 1 ? "" : "s"))", lm, lags, h0: "no serial correlation of the errors up to order \(lags)",161                     assumptions: "Observations in time order; valid with lagged dependent variables among the regressors; large-sample test.")162        return withF(t, f, lags, d2)163    }164165    /// Sample autocorrelations r₁…r_maxLag (mean removed, denominator Σ(xₜ − x̄)²).166    private static func autocorrelations(_ x: [Double], _ maxLag: Int) -> [Double] {167        let n = x.count168        let m = Descriptive.mean(x)169        let d = x.map { $0 - m }170        let c0 = d.reduce(0) { $0 + $1 * $1 }171        return (1...maxLag).map { k in172            var s = 0.0173            for t in k..<n { s += d[t] * d[t - k] }174            return s / c0175        }176    }177178    /// Ljung–Box Q = n(n + 2) Σₖ rₖ²/(n − k) (or Box–Pierce Q = n Σ rₖ²) on a series, χ²(lags − modelDF).179    /// - Parameter modelDF: number of ARMA parameters fitted to produce the residuals (0 for raw data).180    public static func ljungBox(_ x: [Double], lags: Int, modelDF: Int = 0, boxPierce: Bool = false) throws -> RegressionDiagnostic {181        let n = x.count182        guard lags >= 1, lags < n else { throw StatError.invalidInput("Ljung–Box needs 1 ≤ lags < n") }183        guard lags > modelDF, modelDF >= 0 else { throw StatError.invalidInput("lags must exceed modelDF") }184        let r = autocorrelations(x, lags)185        let nd = Double(n)186        var q = 0.0187        for (i, rk) in r.enumerated() { q += boxPierce ? rk * rk : rk * rk / (nd - Double(i + 1)) }188        q *= boxPierce ? nd : nd * (nd + 2)189        return chi2(boxPierce ? "Box–Pierce (\(lags) lags)" : "Ljung–Box (\(lags) lags)", q, lags - modelDF,190                    h0: "the first \(lags) autocorrelations are all zero (white noise)",191                    assumptions: "Stationary series in time order; large-sample χ² approximation\(boxPierce ? " (Ljung–Box is preferable in small samples)" : "").")192    }193194    // MARK: Normality195196    /// Jarque–Bera normality test JB = n/6·(S² + (K − 3)²/4) ~ χ²(2), with the population (biased)197    /// moment estimators S = m₃/m₂^{3/2}, K = m₄/m₂² about the sample mean.198    public static func jarqueBera(_ x: [Double]) throws -> RegressionDiagnostic {199        let n = x.count200        guard n >= 3 else { throw StatError.insufficientData("Jarque–Bera needs at least 3 observations") }201        let m = Descriptive.mean(x)202        var m2 = 0.0, m3 = 0.0, m4 = 0.0203        for v in x { let d = v - m; m2 += d * d; m3 += d * d * d; m4 += d * d * d * d }204        let nd = Double(n)205        m2 /= nd; m3 /= nd; m4 /= nd206        guard m2 > 0 else { throw StatError.invalidInput("the series is constant") }207        let skew = m3 / pow(m2, 1.5), kurt = m4 / (m2 * m2)208        var t = chi2("Jarque–Bera", nd / 6 * (skew * skew + (kurt - 3) * (kurt - 3) / 4), 2, h0: "the data are normally distributed (skewness 0, kurtosis 3)",209                     assumptions: "Independent observations; asymptotic χ² approximation, unreliable in small samples.")210        t.alternativeStatistic = nil211        return t212    }213214    /// Jarque–Bera test on the residuals of a model.215    public static func jarqueBera(_ model: LinearModelResult) throws -> RegressionDiagnostic { try jarqueBera(model.whitened.e) }216217    /// Sample skewness and kurtosis (population estimators, kurtosis not excess) used by the Jarque–Bera test.218    public static func residualMoments(_ x: [Double]) -> (skewness: Double, kurtosis: Double) {219        let m = Descriptive.mean(x)220        var m2 = 0.0, m3 = 0.0, m4 = 0.0221        for v in x { let d = v - m; m2 += d * d; m3 += d * d * d; m4 += d * d * d * d }222        let nd = Double(x.count)223        m2 /= nd; m3 /= nd; m4 /= nd224        return (m3 / pow(m2, 1.5), m4 / (m2 * m2))225    }226227    // MARK: Functional form228229    /// Ramsey RESET: adds ŷ², …, ŷ^power to the regression and tests their joint significance.230    /// Primary statistic F(q, n − k − q) with q = power − 1; the alternative form is the Wald χ²(q) = q·F231    /// (statsmodels `linear_reset(test_type='fitted')` with `use_f=True` / `False`).232    public static func reset(_ model: LinearModelResult, power: Int = 3) throws -> RegressionDiagnostic {233        guard power >= 2 else { throw StatError.invalidInput("RESET needs power ≥ 2") }234        guard model.weights == nil else { throw StatError.invalidInput("RESET is implemented for OLS models") }235        let n = model.nobs, k = model.names.count, q = power - 1236        guard n > k + q else { throw StatError.insufficientData("not enough observations for the RESET regression") }237        var aug = Matrix(rows: n, cols: k + q)238        for i in 0..<n {239            for j in 0..<k { aug[i, j] = model.design[i, j] }240            var p = model.fitted[i]241            for j in 0..<q { p *= model.fitted[i]; aug[i, k + j] = p }242        }243        // scale the powers to unit norm: the test is invariant, the QR is better conditioned244        for j in k..<(k + q) {245            var norm = 0.0246            for i in 0..<n { norm += aug[i, j] * aug[i, j] }247            norm = norm.squareRoot()248            if norm > 0 { for i in 0..<n { aug[i, j] /= norm } }249        }250        let core = try RegressionMath.solve(aug, model.y)251        let d2 = n - k - q252        let f = ((model.ssr - core.ssr) / Double(q)) / (core.ssr / Double(d2))253        var t = RegressionDiagnostic(name: "Ramsey RESET (powers 2…\(power) of fitted values)", statistic: f, distribution: "F(\(q), \(d2))",254                                     df: [Double(q), Double(d2)], pValue: FisherFDistribution(d1: Double(q), d2: Double(d2)).sf(f), tail: upper,255                                     nullHypothesis: "the linear functional form is adequate (powers of the fitted values add nothing)",256                                     assumptions: "Homoskedastic errors for the classical F form; detects neglected non-linearity, not omitted variables in general.")257        t.alternativeStatistic = f * Double(q)258        t.alternativeDistribution = "chi2(\(q))"259        t.alternativePValue = ChiSquaredDistribution(df: Double(q)).sf(f * Double(q))260        return t261    }262}263
Files main
Menu

Theme