1"""F1-conf-03: information-respecting LPM of next-year default on leverage.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Reads only data/firm_panel.csv; writes out/results.json, out/analytic.csv, out/exclusions.csv."""4import json, os5import numpy as np6import pandas as pd7import statsmodels.api as sm89IN = "data/firm_panel.csv"10OUT = "out"11os.makedirs(OUT, exist_ok=True)1213raw = pd.read_csv(IN, dtype={"firm_id": str})14n_raw = len(raw)15ledger, issues = [], []1617def step(name, rule, before, after, why):18 ledger.append(dict(step=name, rule=rule, n_before=before, n_after=after,19 n_removed=before - after, rationale=why))2021# Step 1: column exclusion (information rule) -- no rows removed22excluded = [{"variable": "rating_2025",23 "reason": "Final-vintage field assigned in the 2025 review using realised default history "24 "through 2022 and average sample leverage; known only in 2025, not at end of origin "25 "year t. Including it would leak the outcome (look-ahead)."}]26df = raw.drop(columns=["rating_2025"])27step("1_drop_rating_2025", "drop column rating_2025 (known 2025 > end of year t)", n_raw, len(df),28 "Information-availability rule; column removal, no rows removed")29issues.append({"issue": "Column rating_2025 is a final-vintage (2025) field built from realised defaults "30 "and average leverage: outcome leakage / look-ahead",31 "handling": "Excluded from analytic dataset and model; listed in variables_excluded"})3233# Step 2: exact duplicate rows (assembly error)34n_dup_exact = int(df.duplicated().sum())35key_dup_before = int(raw.duplicated(["firm_id", "year"]).sum())36b = len(df)37df = df.drop_duplicates().reset_index(drop=True)38step("2_drop_exact_duplicates", "drop exact duplicate rows (all columns identical), keep first", b, len(df),39 "Assembly error: same firm-year repeated with identical values; one intended row per firm-year")40issues.append({"issue": f"{key_dup_before} duplicated (firm_id, year) rows in the extract; all are exact "41 f"duplicates of another row ({n_dup_exact} exact duplicates)",42 "handling": "Kept one copy of each firm-year (drop_duplicates)"})43assert not df.duplicated(["firm_id", "year"]).any(), "conflicting firm-year duplicates remain"4445# Step 3: unobserved outcome46b = len(df)47n_na_by_year = df[df.default_next.isna()].year.value_counts().to_dict()48df = df[df.default_next.notna()].copy()49step("3_drop_missing_outcome", "drop rows with default_next missing (not imputed)", b, len(df),50 "Outcome year not observed at data cut (end 2022); missing by year: "51 + ", ".join(f"{k}:{v}" for k, v in sorted(n_na_by_year.items())))52issues.append({"issue": "default_next missing for "53 + ", ".join(f"{k}: {v} rows" for k, v in sorted(n_na_by_year.items()))54 + " (outcome year 2023 beyond data cut)",55 "handling": "Excluded from estimation sample; outcomes not imputed"})5657# Step 4: missing predictors (check)58preds = ["leverage", "cash_ratio", "log_sales"]59b = len(df)60df = df.dropna(subset=preds)61step("4_drop_missing_predictors", "drop rows with any missing predictor", b, len(df),62 "Complete-case requirement for OLS (none expected)")63df["default_next"] = df["default_next"].astype(int)64df = df.sort_values(["firm_id", "year"]).reset_index(drop=True)6566# Estimation: OLS LPM, firm-clustered SE67X = sm.add_constant(df[preds])68fit = sm.OLS(df["default_next"], X).fit(cov_type="cluster", cov_kwds={"groups": df["firm_id"]})6970df[["firm_id", "year"] + preds + ["default_next"]].to_csv(f"{OUT}/analytic.csv", index=False)71pd.DataFrame(ledger).to_csv(f"{OUT}/exclusions.csv", index=False)7273res = {74 "task_id": "F1-conf-03",75 "n_analysed": int(len(df)),76 "n_firms": int(df.firm_id.nunique()),77 "predictors_used": preds,78 "variables_excluded": excluded,79 "beta_leverage": float(fit.params["leverage"]),80 "se_leverage_cluster": float(fit.bse["leverage"]),81 "r2": float(fit.rsquared),82 "default_rate": float(df.default_next.mean()),83 "feasibility": "complete",84 "feasibility_reason": None,85 "data_issues": issues,86 "n_input_rows": {"firm_panel.csv": int(n_raw)},87 "random_seed": None,88 # supplementary (not required by TASK.md)89 "supp_ci95_leverage_cluster": [float(x) for x in fit.conf_int().loc["leverage"]],90 "supp_se_leverage_hc1": float(sm.OLS(df["default_next"], X).fit(cov_type="HC1").bse["leverage"]),91 "supp_beta_cash_ratio": float(fit.params["cash_ratio"]),92 "supp_beta_log_sales": float(fit.params["log_sales"]),93 "supp_intercept": float(fit.params["const"]),94 "supp_year_range": [int(df.year.min()), int(df.year.max())],95}96with open(f"{OUT}/results.json", "w") as f:97 json.dump(res, f, indent=2)98print(json.dumps({k: res[k] for k in ["n_analysed", "n_firms", "beta_leverage",99 "se_leverage_cluster", "r2", "default_rate"]}, indent=2))100