1"""F1-conf-03 analysis: builds the information-respecting analytic dataset and fits the clustered LPM.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py"""3import json, os4import numpy as np, pandas as pd5import statsmodels.api as sm67ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))8OUT = os.path.join(ROOT, "out")9raw = pd.read_csv(os.path.join(ROOT, "data", "firm_panel.csv"), dtype={"firm_id": str})10n_raw = len(raw)11PRED = ["leverage", "cash_ratio", "log_sales"]12ledger = []13def step(name, rule, before, after, why):14 ledger.append(dict(step=name, rule=rule, n_before=before, n_after=after, n_removed=before - after, rationale=why))1516# 1. drop leaking column (column removal, no rows removed)17d = raw.drop(columns=["rating_2025"])18step("1_drop_rating_2025", "drop column rating_2025 (known 2025, built from realised defaults through 2022)",19 n_raw, len(d), "violates information-availability rule; column removal only")20# 2. exact duplicate rows21n_exact_dup = int(raw.duplicated().sum())22n_key_dup = int(raw.duplicated(["firm_id", "year"]).sum())23b = len(d); d = d.loc[~raw.duplicated()].copy()24step("2_dedup", "remove exact duplicate rows (identical firm_id, year and all values in the original file); keep first",25 b, len(d), "assembly error: same firm-year listed twice")26assert not d.duplicated(["firm_id", "year"]).any()27# 3. unobserved outcome28n_missing_y = int(d.default_next.isna().sum())29years_missing = sorted(int(y) for y in d.loc[d.default_next.isna(), "year"].unique())30b = len(d); d = d.dropna(subset=["default_next"]).copy()31step("3_outcome_observed", "drop firm-years with default_next empty", b, len(d),32 f"outcome year not observed at data cut (origin years {years_missing}); no imputation")33# 4. missing predictors (none expected)34b = len(d); d = d.dropna(subset=PRED).copy()35step("4_complete_predictors", "drop rows with missing leverage/cash_ratio/log_sales", b, len(d), "complete-case for regressors")3637d["default_next"] = d["default_next"].astype(int)38d = d.sort_values(["firm_id", "year"]).reset_index(drop=True)39d[["firm_id", "year"] + PRED + ["default_next"]].to_csv(os.path.join(OUT, "analytic.csv"), index=False)40pd.DataFrame(ledger).to_csv(os.path.join(OUT, "exclusions.csv"), index=False)4142# Baseline: unconditional default rate43default_rate = float(d.default_next.mean())44# Model45X = sm.add_constant(d[PRED]); y = d.default_next46codes = pd.factorize(d.firm_id)[0]47fit = sm.OLS(y, X).fit(cov_type="cluster", cov_kwds={"groups": codes})48fit_naive = sm.OLS(y, X).fit()4950res = {51 "task_id": "F1-conf-03",52 "n_analysed": int(len(d)),53 "n_firms": int(d.firm_id.nunique()),54 "predictors_used": PRED,55 "variables_excluded": [56 {"variable": "rating_2025", "reason": "Final-vintage field assigned in 2025 using realised default history through 2022 and sample-average leverage; not known at end of origin year t (outcome leakage)."},57 ],58 "beta_leverage": float(fit.params["leverage"]),59 "se_leverage_cluster": float(fit.bse["leverage"]),60 "r2": float(fit.rsquared),61 "default_rate": default_rate,62 "feasibility": "complete",63 "feasibility_reason": None,64 "data_issues": [65 {"issue": f"{n_exact_dup} exact duplicate rows ({n_key_dup} duplicated firm_id-year keys, all with identical values)",66 "handling": "kept one copy of each; after dedup the panel is balanced (300 firms x 7 years)"},67 {"issue": f"{n_missing_y} unique firm-years with empty default_next (origin years {years_missing}; outcome year not observed by end-2022 data cut)",68 "handling": "excluded from estimation, not imputed; listed in exclusions.csv"},69 {"issue": "rating_2025 is a post-period (2025) field derived from realised defaults: information leakage",70 "handling": "excluded from predictors and from analytic.csv"},71 ],72 "n_input_rows": {"firm_panel.csv": int(n_raw)},73 "random_seed": None,74 # supplementary (not required)75 "supp_ci95_leverage_cluster": [float(x) for x in fit.conf_int().loc["leverage"]],76 "supp_pvalue_leverage_cluster": float(fit.pvalues["leverage"]),77 "supp_se_leverage_iid": float(fit_naive.bse["leverage"]),78 "supp_coef_cash_ratio": float(fit.params["cash_ratio"]),79 "supp_coef_log_sales": float(fit.params["log_sales"]),80 "supp_intercept": float(fit.params["const"]),81 "supp_n_defaults": int(d.default_next.sum()),82}83with open(os.path.join(OUT, "results.json"), "w") as f:84 json.dump(res, f, indent=2)85print(json.dumps({k: res[k] for k in ["n_analysed", "n_firms", "beta_leverage", "se_leverage_cluster", "r2", "default_rate"]}, indent=2))86