1"""F1-conf-03: information-respecting LPM of next-year default on leverage.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py"""3import json, os4import numpy as np, pandas as pd5import statsmodels.api as sm67OUT = "out"8os.makedirs(OUT, exist_ok=True)9raw = pd.read_csv("data/firm_panel.csv", dtype={"firm_id": str})10n_raw = len(raw)11ledger = []12def step(name, rule, before, after, why):13 ledger.append(dict(step=name, rule=rule, n_before=before, n_after=after,14 n_removed=before - after, rationale=why))1516d = raw.copy()17# 1. drop leakage column (column removal, no rows removed)18d = d.drop(columns=["rating_2025"])19step("1_drop_rating_2025", "remove column rating_2025 (known 2025, uses realised defaults)",20 n_raw, len(d), "violates information-availability rule (post-origin, outcome-derived)")21# 2. exact duplicate rows22b = len(d); n_keydup = int(raw.duplicated(["firm_id", "year"]).sum())23d = d.drop_duplicates().reset_index(drop=True)24step("2_drop_exact_duplicates", "drop exact duplicate firm-year rows (keep first)", b, len(d),25 "assembly error: identical repeated firm-year records would double-count")26assert not d.duplicated(["firm_id", "year"]).any()27n_dup_removed = b - len(d)28# 3. unobserved outcome29b = len(d); miss_years = sorted(d.loc[d.default_next.isna(), "year"].unique().tolist())30d = d[d.default_next.notna()].copy()31step("3_drop_missing_outcome", "drop rows with default_next missing", b, len(d),32 "outcome not observed at data cut (end 2022); outcomes are not imputed")33# 4. missing predictors34preds = ["leverage", "cash_ratio", "log_sales"]35b = len(d); d = d.dropna(subset=preds)36step("4_drop_missing_predictors", "drop rows with any missing predictor", b, len(d), "complete-case OLS")37d["default_next"] = d["default_next"].astype(int)38d = d.sort_values(["firm_id", "year"]).reset_index(drop=True)3940X = sm.add_constant(d[preds]); y = d["default_next"]41codes = pd.factorize(d["firm_id"])[0]42fit = sm.OLS(y, X).fit(cov_type="cluster", cov_kwds={"groups": codes})43n_floor = int((d.leverage == d.leverage.min()).sum())4445res = {46 "task_id": "F1-conf-03",47 "n_analysed": int(len(d)),48 "n_firms": int(d.firm_id.nunique()),49 "predictors_used": preds,50 "variables_excluded": [51 {"variable": "rating_2025", "reason": "known only in 2025 and constructed from the firm's realised default history through 2022 (contains the outcome) and average leverage over the sample; violates the end-of-year-t availability rule (look-ahead/target leakage)"},52 {"variable": "default_next", "reason": "outcome variable, not a predictor (known only at end of t+1)"},53 {"variable": "firm_id", "reason": "identifier; used only for clustering, not as a regressor"},54 {"variable": "year", "reason": "origin-year identifier; not part of the specified model"},55 ],56 "beta_leverage": float(fit.params["leverage"]),57 "se_leverage_cluster": float(fit.bse["leverage"]),58 "r2": float(fit.rsquared),59 "default_rate": float(y.mean()),60 "feasibility": "complete",61 "feasibility_reason": None,62 "data_issues": [63 {"issue": f"{n_dup_removed} exact duplicate firm-year rows ({n_keydup} duplicated firm_id-year keys, all with identical values)", "handling": "dropped duplicates keeping one copy; panel becomes balanced 300 firms x 7 years"},64 {"issue": f"default_next missing for {int(raw.default_next.isna().sum())} raw rows, all in origin year(s) {miss_years} (outcome year not observed at data cut)", "handling": "rows excluded from estimation (ledger step 3); no imputation"},65 {"issue": "rating_2025 is a final-vintage field built from realised defaults (outcome leakage)", "handling": "column excluded from analytic dataset"},66 {"issue": f"leverage has a mass point at its minimum 0.02 ({n_floor} firm-years in estimation sample), suggesting floor/censoring in the extract", "handling": "kept as recorded; noted as limitation"},67 ],68 "n_input_rows": {"firm_panel.csv": int(n_raw)},69 "random_seed": None,70 "baseline_default_rate": float(y.mean()),71 "intercept": float(fit.params["const"]),72 "beta_cash_ratio": float(fit.params["cash_ratio"]),73 "beta_log_sales": float(fit.params["log_sales"]),74 "se_leverage_classical": float(sm.OLS(y, X).fit().bse["leverage"]),75 "ci95_leverage_cluster": [float(v) for v in fit.conf_int().loc["leverage"]],76 "p_leverage_cluster": float(fit.pvalues["leverage"]),77 "n_clusters": int(len(np.unique(codes))),78}79d[["firm_id", "year"] + preds + ["default_next"]].to_csv(f"{OUT}/analytic.csv", index=False)80pd.DataFrame(ledger).to_csv(f"{OUT}/exclusions.csv", index=False)81with open(f"{OUT}/results.json", "w") as f:82 json.dump(res, f, indent=2)83print(f"n={res['n_analysed']} beta_leverage={res['beta_leverage']:.6f} se={res['se_leverage_cluster']:.6f} r2={res['r2']:.6f} default_rate={res['default_rate']:.6f}")84