1"""F1-conf-03: information-respecting LPM of next-year default on leverage.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py"""3import json, os4import numpy as np, pandas as pd5import statsmodels.api as sm67IN = "data/firm_panel.csv"; OUT = "out"8os.makedirs(OUT, exist_ok=True)9raw = pd.read_csv(IN, dtype={"firm_id": str})10n_raw = len(raw)11led = []12def step(name, rule, before, after, why):13 led.append(dict(step=name, rule=rule, n_before=before, n_after=after, n_removed=before - after, rationale=why))1415# Step 1: drop leaking column (no rows removed)16df = raw.drop(columns=["rating_2025"])17step("1_drop_rating_2025", "remove column rating_2025 (known only in 2025)", n_raw, len(df),18 "Final-vintage rating built from realised defaults through 2022; not known at end of t (look-ahead leakage).")19# Step 2: exact duplicate rows20b = len(df)21dup_full = int(raw.duplicated().sum())22df = df.loc[~raw.duplicated()].copy()23assert not df.duplicated(["firm_id", "year"]).any()24step("2_drop_exact_duplicates", "drop rows identical on all columns to an earlier row", b, len(df),25 "Assembly error: duplicated firm-year records; keep one copy per firm-year.")26# Step 3: unobserved outcome27b = len(df)28n_missing_y = int(df["default_next"].isna().sum())29years_missing = sorted(df.loc[df["default_next"].isna(), "year"].unique().tolist())30df = df.dropna(subset=["default_next"]).copy()31step("3_drop_unobserved_outcome", "drop rows with empty default_next", b, len(df),32 "Outcome year beyond data cut (end 2022); outcomes are not imputed.")33# Step 4: missing predictors (none expected)34preds = ["leverage", "cash_ratio", "log_sales"]35b = len(df)36df = df.dropna(subset=preds).copy()37step("4_drop_missing_predictors", "drop rows with any missing predictor", b, len(df), "Complete-case for regressors.")3839df["default_next"] = df["default_next"].astype(int)40df = df.sort_values(["firm_id", "year"]).reset_index(drop=True)41df[["firm_id", "year"] + preds + ["default_next"]].to_csv(f"{OUT}/analytic.csv", index=False)42pd.DataFrame(led).to_csv(f"{OUT}/exclusions.csv", index=False)4344X = sm.add_constant(df[preds]); y = df["default_next"]45groups = pd.factorize(df["firm_id"])[0]46fit = sm.OLS(y, X).fit(cov_type="cluster", cov_kwds={"groups": groups})47fit_hc = sm.OLS(y, X).fit(cov_type="HC1")4849res = {50 "task_id": "F1-conf-03",51 "n_analysed": int(len(df)),52 "n_firms": int(df["firm_id"].nunique()),53 "predictors_used": preds,54 "variables_excluded": [55 {"variable": "rating_2025", "reason": "Known only in 2025 and constructed from realised default history through 2022 and sample-average leverage; violates end-of-year-t availability rule (outcome leakage)."},56 {"variable": "default_next", "reason": "Outcome variable (known end of t+1); dependent variable, not a predictor."},57 {"variable": "firm_id", "reason": "Identifier; used only for clustering standard errors, not as a regressor."},58 {"variable": "year", "reason": "Origin-year index; not part of the specified model, used only to identify firm-years."},59 ],60 "beta_leverage": float(fit.params["leverage"]),61 "se_leverage_cluster": float(fit.bse["leverage"]),62 "r2": float(fit.rsquared),63 "default_rate": float(y.mean()),64 "feasibility": "complete",65 "feasibility_reason": None,66 "data_issues": [67 {"issue": f"{dup_full} rows are exact duplicates of another row (duplicated firm_id-year keys, identical on all columns).",68 "handling": "Dropped duplicate copies, keeping one row per firm-year (exclusion step 2)."},69 {"issue": f"default_next empty for {n_missing_y} de-duplicated rows, all in origin year(s) {years_missing} (outcome beyond the end-2022 data cut).",70 "handling": "Excluded from estimation; not imputed (exclusion step 3)."},71 {"issue": "rating_2025 is a final-vintage field built from realised defaults (all defaults carry rating D) - look-ahead leakage.",72 "handling": "Column removed from analytic dataset and listed in variables_excluded (exclusion step 1)."},73 {"issue": "Some firms have default_next = 1 in several years (default not absorbing in the panel).",74 "handling": "Kept as recorded; codebook does not define default as absorbing. Noted as a limitation."},75 {"issue": "leverage bounded exactly at 0.02 and 0.95 (possible truncation in the source).",76 "handling": "Kept as recorded; noted as a limitation."},77 ],78 "n_input_rows": {"firm_panel.csv": int(n_raw)},79 "random_seed": None,80 "supplementary": {81 "intercept": float(fit.params["const"]),82 "beta_cash_ratio": float(fit.params["cash_ratio"]),83 "beta_log_sales": float(fit.params["log_sales"]),84 "se_cash_ratio_cluster": float(fit.bse["cash_ratio"]),85 "se_log_sales_cluster": float(fit.bse["log_sales"]),86 "ci95_leverage_cluster": [float(v) for v in fit.conf_int().loc["leverage"]],87 "p_leverage_cluster": float(fit.pvalues["leverage"]),88 "se_leverage_hc1": float(fit_hc.bse["leverage"]),89 "n_clusters": int(len(np.unique(groups))),90 "cluster_se_note": "statsmodels cluster-robust covariance with default small-sample correction (G/(G-1))*((N-1)/(N-K)); CI uses normal critical values.",91 },92}93with open(f"{OUT}/results.json", "w") as f:94 json.dump(res, f, indent=2)95print(f"n={res['n_analysed']} firms={res['n_firms']} beta_lev={res['beta_leverage']:.4f} se={res['se_leverage_cluster']:.4f} r2={res['r2']:.4f} rate={res['default_rate']:.4f}")96