1"""F1-conf-03: LPM of next-year default on end-of-year-t predictors. Run from workspace root."""2import json, os3import numpy as np, pandas as pd4import statsmodels.api as sm56os.makedirs("out", exist_ok=True)7raw = pd.read_csv("data/firm_panel.csv", dtype={"firm_id": str})8n_raw = len(raw)9PRED = ["leverage", "cash_ratio", "log_sales"]10ledger = []11def step(name, rule, before, after, why):12 ledger.append(dict(step=name, rule=rule, n_before=before, n_after=after, n_removed=before - after, rationale=why))1314step("0_read", "rows read from data/firm_panel.csv", n_raw, n_raw, "raw input")15n_exact_dup = int(raw.duplicated().sum())16n_key_dup = int(raw.duplicated(["firm_id", "year"]).sum())17d = raw.drop_duplicates().reset_index(drop=True)18assert not d.duplicated(["firm_id", "year"]).any(), "non-identical key duplicates present"19step("1_dedupe", "drop exact duplicate rows (keep first)", n_raw, len(d), "assembly error: identical firm-year rows repeated")20step("2_drop_rating_2025", "remove column rating_2025 (known 2025, uses realised defaults)", len(d), len(d), "violates information-availability rule; column removal only")21n_b = len(d)22n_miss_by_year = d.loc[d.default_next.isna(), "year"].value_counts().to_dict()23d = d[d.default_next.notna()].copy()24step("3_outcome_observed", "drop rows with missing default_next", n_b, len(d), "outcome year not observed at data cut (end 2022); no imputation")25n_b = len(d)26d = d.dropna(subset=PRED)27step("4_complete_predictors", "drop rows with missing predictors", n_b, len(d), "listwise deletion (none expected)")2829d["default_next"] = d["default_next"].astype(int)30d = d.sort_values(["firm_id", "year"]).reset_index(drop=True)31analytic = d[["firm_id", "year"] + PRED + ["default_next"]]32analytic.to_csv("out/analytic.csv", index=False)33pd.DataFrame(ledger).to_csv("out/exclusions.csv", index=False)3435X = sm.add_constant(analytic[PRED])36groups = pd.factorize(analytic["firm_id"])[0]37fit = sm.OLS(analytic["default_next"], X).fit(cov_type="cluster", cov_kwds={"groups": groups})3839res = {40 "task_id": "F1-conf-03",41 "n_analysed": int(len(analytic)),42 "n_firms": int(analytic.firm_id.nunique()),43 "predictors_used": PRED,44 "variables_excluded": [45 {"variable": "rating_2025", "reason": "Assigned in the 2025 review using realised default history through 2022 and sample-average leverage; not known at end of year t (look-ahead leakage). In the data every observed default is in rating D."},46 {"variable": "firm_id", "reason": "identifier; used only for clustering, not as a predictor"},47 {"variable": "year", "reason": "origin-year index; not part of the specified model"},48 {"variable": "default_next", "reason": "outcome variable"},49 ],50 "beta_leverage": float(fit.params["leverage"]),51 "se_leverage_cluster": float(fit.bse["leverage"]),52 "r2": float(fit.rsquared),53 "default_rate": float(analytic.default_next.mean()),54 "feasibility": "complete",55 "feasibility_reason": None,56 "data_issues": [57 {"issue": f"{n_exact_dup} exact duplicate rows ({n_key_dup} duplicated firm_id-year keys, all identical copies)", "handling": "dropped duplicates keeping one copy; panel becomes 300 firms x 7 years"},58 {"issue": "rating_2025 is a final-vintage field built from future defaults (information leakage)", "handling": "excluded from predictors and from out/analytic.csv"},59 {"issue": f"default_next missing for {sum(n_miss_by_year.values())} deduplicated rows, by year: {{{', '.join(f'{int(k)}: {v}' for k, v in sorted(n_miss_by_year.items()))}}}", "handling": "excluded from estimation (outcome not observed at data cut); not imputed"},60 ],61 "n_input_rows": {"firm_panel.csv": int(n_raw)},62 "random_seed": None,63 # supplementary64 "beta_cash_ratio": float(fit.params["cash_ratio"]),65 "beta_log_sales": float(fit.params["log_sales"]),66 "intercept": float(fit.params["const"]),67 "ci95_leverage_low": float(fit.conf_int().loc["leverage", 0]),68 "ci95_leverage_high": float(fit.conf_int().loc["leverage", 1]),69 "p_leverage_cluster": float(fit.pvalues["leverage"]),70 "se_leverage_classical": float(sm.OLS(analytic["default_next"], X).fit().bse["leverage"]),71 "n_rows_after_dedupe": int(n_raw - n_exact_dup),72}73with open("out/results.json", "w") as f:74 json.dump(res, f, indent=2)75print(json.dumps({k: res[k] for k in ["n_analysed", "n_firms", "beta_leverage", "se_leverage_cluster", "r2", "default_rate"]}, indent=2))76