1"""F1-conf-03: information-respecting analytic dataset and LPM for next-year default.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py"""3import json, os4import numpy as np, pandas as pd5import statsmodels.formula.api as smf67IN = "data/firm_panel.csv"; OUT = "out"8os.makedirs(OUT, exist_ok=True)9raw = pd.read_csv(IN, dtype={"firm_id": str})10n_read = len(raw)11PRED = ["leverage", "cash_ratio", "log_sales"]12ledger, issues = [], []13def step(rule, before, after, why):14 ledger.append(dict(step=len(ledger) + 1, rule=rule, n_before=before, n_after=after,15 n_removed=before - after, rationale=why))1617# Step 1: rows read18step("rows read from firm_panel.csv", n_read, n_read, "starting point")1920# Step 2: exact duplicate rows (assembly error)21n_exact = int(raw.duplicated().sum())22n_key = int(raw.duplicated(["firm_id", "year"]).sum())23d = raw.drop_duplicates().copy()24assert not d.duplicated(["firm_id", "year"]).any(), "conflicting firm-year duplicates"25step("drop exact duplicate rows (same firm_id-year, identical values)", n_read, len(d),26 "assembly error: rows repeated in extract; one copy kept")27issues.append(dict(issue=f"{n_exact} exact duplicate rows ({n_key} duplicated firm_id-year keys; all copies identical in every column)",28 handling="dropped the redundant copies, kept one row per firm-year (300 firms x 7 years = 2100 rows)"))2930# Step 3: column removal for look-ahead variable31step("exclude column rating_2025 (known only in 2025)", len(d), len(d),32 "final-vintage field built from realised defaults through 2022 and sample-average leverage: look-ahead / outcome leakage")33d = d.drop(columns=["rating_2025"])34issues.append(dict(issue="rating_2025 is a 2025 final-vintage field constructed from the realised default history (only class D has any defaults in the data)",35 handling="excluded from predictors and from analytic.csv"))3637# Step 4: unobserved outcome38before = len(d)39miss_years = sorted(d.loc[d.default_next.isna(), "year"].unique().tolist())40d = d[d.default_next.notna()].copy()41step("drop firm-years with unobserved default_next", before, len(d),42 f"outcome year t+1 not observed at data cut (end 2022); missing outcomes all in year(s) {miss_years}; not imputed")43issues.append(dict(issue=f"default_next missing for {before-len(d)} firm-years (all origin year {miss_years})",44 handling="excluded from estimation sample (right-censored at the data cut), no imputation"))4546# Step 5: missing predictors (none expected)47before = len(d)48d = d.dropna(subset=PRED)49step("drop firm-years with missing predictors", before, len(d), "complete-case requirement for OLS")5051# Plausibility notes (no rows removed)52d["default_next"] = d["default_next"].astype(int)53s = d.sort_values(["firm_id", "year"])54rep = int(((s.groupby("firm_id").default_next.shift() == 1) & (s.default_next == 1)).sum())55issues.append(dict(issue=f"default is not absorbing: firms record default_next=1 in multiple years ({rep} firm-years with default following a prior-year default) and remain in the panel",56 handling="kept as given (default treated as a recurring annual event); noted as limitation"))57issues.append(dict(issue=f"leverage bounded in [{d.leverage.min():.2f}, {d.leverage.max():.2f}], suggesting possible winsorisation/clipping in the simulation",58 handling="kept as given"))5960# Model61m = smf.ols("default_next ~ leverage + cash_ratio + log_sales", data=d).fit(62 cov_type="cluster", cov_kwds={"groups": pd.factorize(d["firm_id"])[0]})63ana = d[["firm_id", "year"] + PRED + ["default_next"]].sort_values(["firm_id", "year"])64ana.to_csv(f"{OUT}/analytic.csv", index=False)65pd.DataFrame(ledger).to_csv(f"{OUT}/exclusions.csv", index=False)6667res = dict(68 task_id="F1-conf-03",69 n_analysed=int(len(ana)), n_firms=int(ana.firm_id.nunique()),70 predictors_used=PRED,71 variables_excluded=[dict(variable="rating_2025",72 reason="known only in 2025; built from realised default history through 2022 and average leverage over the sample (look-ahead / outcome leakage)")],73 beta_leverage=float(m.params["leverage"]),74 se_leverage_cluster=float(m.bse["leverage"]),75 r2=float(m.rsquared),76 default_rate=float(ana.default_next.mean()),77 feasibility="complete", feasibility_reason=None,78 data_issues=issues,79 n_input_rows={"firm_panel.csv": int(n_read)},80 random_seed=None,81 extra=dict(ci95_leverage=[float(x) for x in m.conf_int().loc["leverage"]],82 p_leverage=float(m.pvalues["leverage"]),83 coef=m.params.to_dict(), se_cluster=m.bse.to_dict(),84 n_clusters=int(ana.firm_id.nunique()),85 years=[int(ana.year.min()), int(ana.year.max())]),86)87with open(f"{OUT}/results.json", "w") as f:88 json.dump(res, f, indent=2)89print(json.dumps(res, indent=2)); print(m.summary())90