1"""F1-conf-03: information-respecting LPM of next-year default on leverage.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Reads only data/firm_panel.csv; writes out/results.json, out/analytic.csv, out/exclusions.csv."""4import json, os5import numpy as np6import pandas as pd7import statsmodels.api as sm89IN = "data/firm_panel.csv"10OUT = "out"11os.makedirs(OUT, exist_ok=True)12PRED = ["leverage", "cash_ratio", "log_sales"]1314raw = pd.read_csv(IN, dtype={"firm_id": str})15n_raw = len(raw)16ex = []17def step(name, rule, before, after, why):18 ex.append(dict(step=name, rule=rule, n_before=before, n_after=after,19 n_removed=before - after, rationale=why))2021issues = []22# Step 1: column exclusion (information rule)23d = raw.drop(columns=["rating_2025"])24step("1_drop_column_rating_2025", "drop column rating_2025 (known only in 2025)", n_raw, len(d),25 "Final-vintage field assigned in 2025 using realised default history through 2022 and sample-average leverage; not known at end of year t (look-ahead leakage).")26issues.append(dict(issue="rating_2025 is a final-vintage (2025) field built from realised defaults through 2022 and average sample leverage: it violates the information-availability rule.",27 handling="Column dropped from the analytic dataset; listed in variables_excluded; never used."))2829# Step 2: exact duplicates30n_full_dup = int(raw.duplicated().sum())31n_key_dup = int(raw.duplicated(["firm_id", "year"]).sum())32b = len(d)33d = d[~raw.duplicated()].copy()34step("2_drop_exact_duplicates", "drop rows identical to an earlier row on all 7 columns", b, len(d),35 "Assembly error: firm-year rows repeated verbatim; keeping them would double-count observations.")36issues.append(dict(issue=f"{n_full_dup} exact duplicate rows (same firm_id-year, all fields identical); input has {n_raw} rows vs 2100 intended (300 firms x 7 years).",37 handling="Kept first occurrence, dropped the copies (ledger step 2)."))38assert not d.duplicated(["firm_id", "year"]).any(), "conflicting firm-year duplicates remain"3940# Step 3: unobserved outcome41b = len(d)42miss_years = sorted(d.loc[d.default_next.isna(), "year"].unique().tolist())43d = d[d.default_next.notna()].copy()44step("3_drop_missing_outcome", "drop firm-years with default_next empty", b, len(d),45 f"Outcome year t+1 not observed by the data cut (end 2022); occurs in years {miss_years}. Outcomes not imputed.")46issues.append(dict(issue=f"default_next missing (right-censored at the end-2022 data cut) for origin years {miss_years}.",47 handling="Firm-years excluded from estimation (ledger step 3); no imputation."))4849# Step 4: missing predictors (none expected, recorded for completeness)50b = len(d)51d = d.dropna(subset=PRED)52step("4_drop_missing_predictors", "drop rows with missing leverage/cash_ratio/log_sales", b, len(d),53 "Complete-case requirement for OLS (no rows affected if 0 removed).")5455n_floor = int((d.leverage == d.leverage.min()).sum())56issues.append(dict(issue=f"leverage has a mass point at its minimum value {d.leverage.min()} ({n_floor} analysed firm-years), suggesting a floor/clipping in the source.",57 handling="Kept as recorded; noted as a limitation (no correction applied)."))5859d["default_next"] = d["default_next"].astype(int)60assert set(d.default_next.unique()) <= {0, 1}61d = d.sort_values(["firm_id", "year"]).reset_index(drop=True)62analytic = d[["firm_id", "year"] + PRED + ["default_next"]]63analytic.to_csv(f"{OUT}/analytic.csv", index=False)64pd.DataFrame(ex).to_csv(f"{OUT}/exclusions.csv", index=False)6566X = sm.add_constant(analytic[PRED])67groups = pd.factorize(analytic.firm_id)[0]68fit = sm.OLS(analytic.default_next, X).fit(cov_type="cluster", cov_kwds={"groups": groups})69fit_hc = sm.OLS(analytic.default_next, X).fit(cov_type="HC1")7071res = {72 "task_id": "F1-conf-03",73 "n_analysed": int(len(analytic)),74 "n_firms": int(analytic.firm_id.nunique()),75 "predictors_used": PRED,76 "variables_excluded": [77 {"variable": "rating_2025", "reason": "Known only in 2025; built from realised default history through 2022 and sample-average leverage (look-ahead leakage); violates information-availability rule."},78 {"variable": "default_next", "reason": "Outcome (dependent variable), known only at end of t+1; not a predictor."},79 {"variable": "firm_id", "reason": "Identifier; used only for clustering standard errors, not as a regressor."},80 {"variable": "year", "reason": "Origin-year identifier; not part of the specified model."},81 ],82 "beta_leverage": float(fit.params["leverage"]),83 "se_leverage_cluster": float(fit.bse["leverage"]),84 "r2": float(fit.rsquared),85 "default_rate": float(analytic.default_next.mean()),86 "feasibility": "complete",87 "feasibility_reason": None,88 "data_issues": issues,89 "n_input_rows": {"firm_panel.csv": int(n_raw)},90 "random_seed": None,91 # supplementary (not required)92 "supp_ci95_leverage_cluster": [float(x) for x in fit.conf_int().loc["leverage"]],93 "supp_pvalue_leverage_cluster": float(fit.pvalues["leverage"]),94 "supp_se_leverage_hc1": float(fit_hc.bse["leverage"]),95 "supp_coefficients": {k: float(v) for k, v in fit.params.items()},96 "supp_se_cluster": {k: float(v) for k, v in fit.bse.items()},97 "supp_n_clusters": int(len(np.unique(groups))),98 "supp_years_analysed": [int(analytic.year.min()), int(analytic.year.max())],99 "supp_n_exact_duplicates": n_full_dup,100 "supp_n_key_duplicates": n_key_dup,101}102with open(f"{OUT}/results.json", "w") as f:103 json.dump(res, f, indent=2)104print(f"n_analysed={res['n_analysed']} n_firms={res['n_firms']} beta_leverage={res['beta_leverage']:.4f} "105 f"se_cluster={res['se_leverage_cluster']:.4f} r2={res['r2']:.4f} default_rate={res['default_rate']:.4f}")106