1"""F1-conf-03: information-respecting LPM of next-year default on leverage.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Reads only data/firm_panel.csv; writes out/results.json, out/analytic.csv, out/exclusions.csv.4No randomness is used (random_seed = null)."""5import json, os6import numpy as np7import pandas as pd8import statsmodels.api as sm910DATA = "data/firm_panel.csv"11OUT = "out"12os.makedirs(OUT, exist_ok=True)1314raw = pd.read_csv(DATA, dtype={"firm_id": str})15n_raw = len(raw)16ledger = []17issues = []18def step(name, rule, before, after, why):19 ledger.append(dict(step=name, rule=rule, n_before=before, n_after=after,20 n_removed=before - after, rationale=why))2122step("0_read", "rows read from data/firm_panel.csv (excluding header)", n_raw, n_raw,23 "starting point")2425# 1. Remove look-ahead column (column removal, no rows removed)26PREDICTORS = ["leverage", "cash_ratio", "log_sales"]27excluded = [{"variable": "rating_2025",28 "reason": "Known only in 2025 (final-vintage field) and constructed from the firm's realised "29 "default history through 2022 and sample-average leverage; not available at end of "30 "origin year t and mechanically contains the outcome (look-ahead leakage)."}]31df = raw.drop(columns=["rating_2025"])32step("1_drop_rating_2025", "drop column rating_2025 (known 2025, uses realised outcomes)", len(df), len(df),33 "information-availability rule; column removal only")34_dr = raw.drop_duplicates().groupby("rating_2025")["default_next"].mean()35leak_rate_by_rating = {k: float(v) for k, v in _dr.items()}36issues.append({"issue": "Column rating_2025 is a final-vintage (2025) field built from realised defaults through 2022 "37 f"(look-ahead; observed default rate by rating: {leak_rate_by_rating}).",38 "handling": "Excluded from the analytic dataset and listed in variables_excluded."})3940# 2. Exact duplicate rows41n_keydup = int(raw.duplicated(["firm_id", "year"]).sum())42n_exdup = int(raw.duplicated().sum())43before = len(df)44df = df.loc[~raw.duplicated()].copy() # duplicates judged on all original columns45step("2_drop_exact_duplicates", "drop rows identical on all columns to an earlier row (keep first)", before, len(df),46 "assembly error: one intended row per firm-year; duplicates would double-weight observations")47assert not df.duplicated(["firm_id", "year"]).any(), "conflicting firm-year duplicates remain"48issues.append({"issue": f"{n_exdup} exact duplicate rows ({n_keydup} duplicated firm_id-year keys; all duplicates identical "49 f"on every column, no conflicting values).",50 "handling": "Kept the first occurrence of each, giving one row per firm-year (300 firms x 7 years)."})5152# 3. Missing outcome53n_miss = int(df["default_next"].isna().sum())54miss_years = sorted(df.loc[df["default_next"].isna(), "year"].unique().tolist())55before = len(df)56df = df.loc[df["default_next"].notna()].copy()57step("3_drop_missing_outcome", "drop firm-years with default_next empty (outcome year not observed by end-2022 cut)",58 before, len(df), "outcome not observed; no imputation allowed")59issues.append({"issue": f"{n_miss} firm-years (after de-duplication) have missing default_next, all in origin year(s) {miss_years} "60 "(outcome year 2023 beyond the data cut).",61 "handling": "Excluded from estimation (no imputation); recorded in the exclusion ledger."})6263# 4. Missing predictors (none expected; ledger step kept for completeness)64before = len(df)65df = df.dropna(subset=PREDICTORS)66step("4_drop_missing_predictors", "drop rows with missing leverage, cash_ratio or log_sales", before, len(df),67 "complete-case for regressors")6869df["default_next"] = df["default_next"].astype(int)70df = df.sort_values(["firm_id", "year"]).reset_index(drop=True)71df[["firm_id", "year"] + PREDICTORS + ["default_next"]].to_csv(f"{OUT}/analytic.csv", index=False)72pd.DataFrame(ledger).to_csv(f"{OUT}/exclusions.csv", index=False)7374# Baseline75default_rate = float(df["default_next"].mean())7677# Model: OLS LPM, SE clustered by firm78X = sm.add_constant(df[PREDICTORS])79y = df["default_next"]80groups = pd.factorize(df["firm_id"])[0]81fit = sm.OLS(y, X).fit(cov_type="cluster", cov_kwds={"groups": groups})82b = float(fit.params["leverage"]); se = float(fit.bse["leverage"])8384res = {85 "task_id": "F1-conf-03",86 "n_analysed": int(len(df)),87 "n_firms": int(df["firm_id"].nunique()),88 "predictors_used": PREDICTORS,89 "variables_excluded": excluded,90 "beta_leverage": b,91 "se_leverage_cluster": se,92 "r2": float(fit.rsquared),93 "default_rate": default_rate,94 "feasibility": "complete",95 "feasibility_reason": None,96 "data_issues": issues,97 "n_input_rows": {"firm_panel.csv": int(n_raw)},98 "random_seed": None,99 # supplementary (not required)100 "ci95_leverage_cluster": [float(fit.conf_int().loc["leverage", 0]), float(fit.conf_int().loc["leverage", 1])],101 "pvalue_leverage_cluster": float(fit.pvalues["leverage"]),102 "beta_cash_ratio": float(fit.params["cash_ratio"]),103 "se_cash_ratio_cluster": float(fit.bse["cash_ratio"]),104 "beta_log_sales": float(fit.params["log_sales"]),105 "se_log_sales_cluster": float(fit.bse["log_sales"]),106 "intercept": float(fit.params["const"]),107 "default_rate_by_rating_2025_diagnostic": leak_rate_by_rating,108 "n_clusters": int(len(np.unique(groups))),109 "cov_type": "cluster-robust by firm_id (statsmodels default small-sample correction)",110}111with open(f"{OUT}/results.json", "w") as f:112 json.dump(res, f, indent=2)113print(f"n_analysed={res['n_analysed']} n_firms={res['n_firms']} beta_leverage={b:.6f} se={se:.6f} "114 f"r2={res['r2']:.6f} default_rate={default_rate:.6f}")115