1"""F1-conf-03: information-respecting analytic dataset and LPM for next-year default.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py"""3import json, os4import pandas as pd5import statsmodels.formula.api as smf67os.makedirs("out", exist_ok=True)8raw = pd.read_csv("data/firm_panel.csv", dtype={"firm_id": str})9n_read = len(raw)10PRED = ["leverage", "cash_ratio", "log_sales"]11ledger, issues = [], []1213def step(rule, before, after, why):14 ledger.append(dict(step=len(ledger) + 1, rule=rule, n_before=before, n_after=after,15 n_removed=before - after, rationale=why))1617step("rows read", n_read, n_read, "data rows in data/firm_panel.csv (header excluded)")1819# 1. column removal: post-origin (look-ahead) variable20step("drop column rating_2025", n_read, n_read,21 "rating_2025 is known only in 2025 and was built from realised defaults through 2022 and "22 "full-sample average leverage: look-ahead/outcome leakage; column removed, no rows removed")23d = raw.drop(columns=["rating_2025"])2425# 2. exact duplicate rows26n_dup = int(raw.duplicated().sum())27n_key_dup = int(raw.duplicated(["firm_id", "year"]).sum())28d = d.loc[~raw.duplicated()].copy()29assert not d.duplicated(["firm_id", "year"]).any()30step("drop exact duplicate rows", n_read, len(d),31 "identical copies of the same firm-year (assembly error); first copy kept")32issues.append(dict(issue=f"{n_dup} exact duplicate rows ({n_key_dup} duplicated firm_id-year keys; all "33 "duplicates identical in every column), giving 40 firms 8 rows instead of 7",34 handling="dropped duplicate copies, keeping one row per firm-year"))3536# 3. missing outcome37before = len(d)38miss = d["default_next"].isna()39miss_years = sorted(d.loc[miss, "year"].unique().tolist())40d = d.loc[~miss].copy()41step("drop rows with missing default_next", before, len(d),42 f"outcome not observed (years {miss_years}: t+1 after the 2022 data cut); not imputed")43issues.append(dict(issue=f"{int(miss.sum())} firm-years with missing default_next, all in year(s) {miss_years} "44 "(outcome year 2023 not observed at end-2022 cut)",45 handling="excluded from estimation and recorded in the exclusion ledger; no imputation"))4647# 4. missing predictors (none expected)48before = len(d)49d = d.dropna(subset=PRED)50step("drop rows with missing predictors", before, len(d), "complete-case on leverage, cash_ratio, log_sales")5152issues.append(dict(issue="rating_2025 is a final-vintage (2025) field derived from realised defaults and average "53 "leverage; it perfectly separates defaulters (all defaults have rating D)",54 handling="excluded from the analytic dataset and the model (information rule)"))55n_floor = int((d["leverage"] == d["leverage"].min()).sum())56issues.append(dict(issue=f"leverage has a mass point at its minimum {d['leverage'].min()} ({n_floor} analysed rows), "57 "suggesting floor-coding/censoring in the extract",58 handling="kept as recorded; noted as a limitation"))59nd = d.groupby("firm_id")["default_next"].sum()60issues.append(dict(issue=f"{int((nd > 1).sum())} firms have default_next = 1 in more than one year (no exit after default)",61 handling="kept; each firm-year treated as an observation as the estimand specifies; firm-clustered SEs"))6263d["default_next"] = d["default_next"].astype(int)64d = d.sort_values(["firm_id", "year"]).reset_index(drop=True)65d[["firm_id", "year"] + PRED + ["default_next"]].to_csv("out/analytic.csv", index=False)66pd.DataFrame(ledger).to_csv("out/exclusions.csv", index=False)6768m = smf.ols("default_next ~ leverage + cash_ratio + log_sales", data=d).fit(69 cov_type="cluster", cov_kwds={"groups": pd.factorize(d["firm_id"])[0]})7071res = dict(72 task_id="F1-conf-03",73 n_analysed=int(len(d)),74 n_firms=int(d["firm_id"].nunique()),75 predictors_used=PRED,76 variables_excluded=[dict(variable="rating_2025",77 reason="known only in 2025 (after origin year t); constructed from realised default "78 "history through 2022 and full-sample average leverage, so it leaks the outcome")],79 beta_leverage=float(m.params["leverage"]),80 se_leverage_cluster=float(m.bse["leverage"]),81 r2=float(m.rsquared),82 default_rate=float(d["default_next"].mean()),83 feasibility="complete",84 feasibility_reason=None,85 data_issues=issues,86 n_input_rows={"firm_panel.csv": int(n_read)},87 random_seed=None,88 # supplementary89 ci95_leverage=[float(x) for x in m.conf_int().loc["leverage"]],90 p_leverage=float(m.pvalues["leverage"]),91 coef_all={k: float(v) for k, v in m.params.items()},92 se_cluster_all={k: float(v) for k, v in m.bse.items()},93)94with open("out/results.json", "w") as f:95 json.dump(res, f, indent=2)96print(json.dumps(res, indent=2))97