1"""F1-conf-03: information-respecting analytic dataset and LPM of next-year default.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Reads only data/firm_panel.csv; writes out/results.json, out/analytic.csv, out/exclusions.csv.4No randomness is used (random_seed = null)."""5import json, os6import numpy as np, pandas as pd7import statsmodels.api as sm89DATA = "data/firm_panel.csv"10OUT = "out"11os.makedirs(OUT, exist_ok=True)1213raw = pd.read_csv(DATA, dtype={"firm_id": str})14n_raw = len(raw)15ledger, issues = [], []1617def step(name, rule, before, after, why):18 ledger.append(dict(step=name, rule=rule, n_before=before, n_after=after,19 n_removed=before - after, rationale=why))2021# Step 1: rows read22step("1_read", "all data rows of firm_panel.csv", n_raw, n_raw, "rows read (excluding header)")2324# Step 2: drop rating_2025 column (information rule)25PREDICTORS = ["leverage", "cash_ratio", "log_sales"]26excluded = [{"variable": "rating_2025",27 "reason": "Final-vintage field assigned in 2025 using realised default history through 2022 "28 "and sample-average leverage; not known at end of origin year t (look-ahead / outcome leakage)."},29 {"variable": "default_next",30 "reason": "Outcome variable (known only at end of year t+1); used as dependent variable, not as a predictor."},31 {"variable": "firm_id",32 "reason": "Identifier; used only for clustering standard errors, not as a predictor."},33 {"variable": "year",34 "reason": "Origin-year index; kept in analytic.csv for identification but not entered as a regressor "35 "(task model specifies leverage, cash_ratio, log_sales only)."}]36df = raw.drop(columns=["rating_2025"])37step("2_drop_column_rating_2025", "remove column rating_2025 (known only in 2025)", len(df), len(df),38 "information-availability rule: value not known at end of year t; column removal, no rows removed")3940# Step 3: exact duplicate rows41n_dup_key = int(raw.duplicated(["firm_id", "year"], keep=False).sum())42n_exact = int(raw.duplicated().sum())43before = len(df)44df = df[~raw.duplicated()].copy()45step("3_drop_exact_duplicates", "drop rows that are exact copies of an earlier row (all columns)", before, len(df),46 "assembly error: duplicated firm-year records; identical in every field so one copy kept")47issues.append({"issue": f"{n_exact} exact duplicate rows ({n_dup_key} rows involved in duplicated firm_id-year keys); "48 "all duplicates identical across all columns",49 "handling": "kept first occurrence, dropped the copies (ledger step 3)"})50assert not df.duplicated(["firm_id", "year"]).any(), "conflicting firm-year duplicates remain"5152# Step 4: unobserved outcome53before = len(df)54miss = df["default_next"].isna()55miss_years = sorted(df.loc[miss, "year"].unique().tolist())56df = df[~miss].copy()57step("4_drop_unobserved_outcome", "drop firm-years with default_next empty", before, len(df),58 f"outcome year not observed at the end-2022 data cut (origin years {miss_years}); outcomes not imputed")59issues.append({"issue": f"default_next missing for {int(miss.sum())} unique firm-years (origin years {miss_years}), "60 "because year t+1 is after the data cut",61 "handling": "excluded from estimation sample, not imputed (ledger step 4)"})6263# Step 5: missing predictors (none expected, but checked)64before = len(df)65df = df.dropna(subset=PREDICTORS)66step("5_drop_missing_predictors", "drop rows with any missing predictor", before, len(df),67 "complete-case requirement for OLS")6869n_floor = int((df["leverage"] == df["leverage"].min()).sum())70issues.append({"issue": f"leverage has a point mass of {n_floor} estimation-sample rows at its minimum "71 f"{df['leverage'].min()} (apparent floor/censoring in the extract)",72 "handling": "retained as recorded; noted as limitation"})73issues.append({"issue": "rating_2025 is a final-vintage field built from realised outcomes (perfectly separates "74 "defaulters: only rating D firms ever default)",75 "handling": "excluded from the analytic dataset (ledger step 2, variables_excluded)"})7677df["default_next"] = df["default_next"].astype(int)78df = df.sort_values(["firm_id", "year"]).reset_index(drop=True)7980# Baseline81default_rate = float(df["default_next"].mean())8283# OLS LPM with firm-clustered SE84X = sm.add_constant(df[PREDICTORS])85y = df["default_next"]86groups = pd.factorize(df["firm_id"])[0]87fit = sm.OLS(y, X).fit(cov_type="cluster", cov_kwds={"groups": groups})88b = float(fit.params["leverage"]); se = float(fit.bse["leverage"])8990df[["firm_id", "year"] + PREDICTORS + ["default_next"]].to_csv(f"{OUT}/analytic.csv", index=False)91pd.DataFrame(ledger).to_csv(f"{OUT}/exclusions.csv", index=False)9293res = {94 "task_id": "F1-conf-03",95 "n_analysed": int(len(df)),96 "n_firms": int(df["firm_id"].nunique()),97 "predictors_used": PREDICTORS,98 "variables_excluded": excluded,99 "beta_leverage": round(b, 6),100 "se_leverage_cluster": round(se, 6),101 "ci95_leverage_cluster": [round(b - 1.959964 * se, 6), round(b + 1.959964 * se, 6)],102 "r2": round(float(fit.rsquared), 6),103 "default_rate": round(default_rate, 6),104 "coef_all": {k: round(float(v), 6) for k, v in fit.params.items()},105 "se_all_cluster": {k: round(float(v), 6) for k, v in fit.bse.items()},106 "se_leverage_nonrobust": round(float(sm.OLS(y, X).fit().bse["leverage"]), 6),107 "cluster_se_type": "statsmodels CR1 (small-sample corrected), G = number of firms",108 "feasibility": "complete",109 "feasibility_reason": None,110 "data_issues": issues,111 "n_input_rows": {"firm_panel.csv": int(n_raw)},112 "random_seed": None,113}114with open(f"{OUT}/results.json", "w") as f:115 json.dump(res, f, indent=2)116print(f"n_analysed={res['n_analysed']} n_firms={res['n_firms']} beta_leverage={res['beta_leverage']} "117 f"se={res['se_leverage_cluster']} r2={res['r2']} default_rate={res['default_rate']}")118