"""F1-conf-03: information-respecting LPM of next-year default on leverage.
Run from workspace root: ~/wp29/venv/bin/python out/analysis.py"""
import json, os
import numpy as np, pandas as pd
import statsmodels.api as sm

IN = "data/firm_panel.csv"; OUT = "out"
os.makedirs(OUT, exist_ok=True)
raw = pd.read_csv(IN, dtype={"firm_id": str})
n_raw = len(raw)
led = []
def step(name, rule, before, after, why):
    led.append(dict(step=name, rule=rule, n_before=before, n_after=after, n_removed=before - after, rationale=why))

# Step 1: drop leaking column (no rows removed)
df = raw.drop(columns=["rating_2025"])
step("1_drop_rating_2025", "remove column rating_2025 (known only in 2025)", n_raw, len(df),
     "Final-vintage rating built from realised defaults through 2022; not known at end of t (look-ahead leakage).")
# Step 2: exact duplicate rows
b = len(df)
dup_full = int(raw.duplicated().sum())
df = df.loc[~raw.duplicated()].copy()
assert not df.duplicated(["firm_id", "year"]).any()
step("2_drop_exact_duplicates", "drop rows identical on all columns to an earlier row", b, len(df),
     "Assembly error: duplicated firm-year records; keep one copy per firm-year.")
# Step 3: unobserved outcome
b = len(df)
n_missing_y = int(df["default_next"].isna().sum())
years_missing = sorted(df.loc[df["default_next"].isna(), "year"].unique().tolist())
df = df.dropna(subset=["default_next"]).copy()
step("3_drop_unobserved_outcome", "drop rows with empty default_next", b, len(df),
     "Outcome year beyond data cut (end 2022); outcomes are not imputed.")
# Step 4: missing predictors (none expected)
preds = ["leverage", "cash_ratio", "log_sales"]
b = len(df)
df = df.dropna(subset=preds).copy()
step("4_drop_missing_predictors", "drop rows with any missing predictor", b, len(df), "Complete-case for regressors.")

df["default_next"] = df["default_next"].astype(int)
df = df.sort_values(["firm_id", "year"]).reset_index(drop=True)
df[["firm_id", "year"] + preds + ["default_next"]].to_csv(f"{OUT}/analytic.csv", index=False)
pd.DataFrame(led).to_csv(f"{OUT}/exclusions.csv", index=False)

X = sm.add_constant(df[preds]); y = df["default_next"]
groups = pd.factorize(df["firm_id"])[0]
fit = sm.OLS(y, X).fit(cov_type="cluster", cov_kwds={"groups": groups})
fit_hc = sm.OLS(y, X).fit(cov_type="HC1")

res = {
    "task_id": "F1-conf-03",
    "n_analysed": int(len(df)),
    "n_firms": int(df["firm_id"].nunique()),
    "predictors_used": preds,
    "variables_excluded": [
        {"variable": "rating_2025", "reason": "Known only in 2025 and constructed from realised default history through 2022 and sample-average leverage; violates end-of-year-t availability rule (outcome leakage)."},
        {"variable": "default_next", "reason": "Outcome variable (known end of t+1); dependent variable, not a predictor."},
        {"variable": "firm_id", "reason": "Identifier; used only for clustering standard errors, not as a regressor."},
        {"variable": "year", "reason": "Origin-year index; not part of the specified model, used only to identify firm-years."},
    ],
    "beta_leverage": float(fit.params["leverage"]),
    "se_leverage_cluster": float(fit.bse["leverage"]),
    "r2": float(fit.rsquared),
    "default_rate": float(y.mean()),
    "feasibility": "complete",
    "feasibility_reason": None,
    "data_issues": [
        {"issue": f"{dup_full} rows are exact duplicates of another row (duplicated firm_id-year keys, identical on all columns).",
         "handling": "Dropped duplicate copies, keeping one row per firm-year (exclusion step 2)."},
        {"issue": f"default_next empty for {n_missing_y} de-duplicated rows, all in origin year(s) {years_missing} (outcome beyond the end-2022 data cut).",
         "handling": "Excluded from estimation; not imputed (exclusion step 3)."},
        {"issue": "rating_2025 is a final-vintage field built from realised defaults (all defaults carry rating D) - look-ahead leakage.",
         "handling": "Column removed from analytic dataset and listed in variables_excluded (exclusion step 1)."},
        {"issue": "Some firms have default_next = 1 in several years (default not absorbing in the panel).",
         "handling": "Kept as recorded; codebook does not define default as absorbing. Noted as a limitation."},
        {"issue": "leverage bounded exactly at 0.02 and 0.95 (possible truncation in the source).",
         "handling": "Kept as recorded; noted as a limitation."},
    ],
    "n_input_rows": {"firm_panel.csv": int(n_raw)},
    "random_seed": None,
    "supplementary": {
        "intercept": float(fit.params["const"]),
        "beta_cash_ratio": float(fit.params["cash_ratio"]),
        "beta_log_sales": float(fit.params["log_sales"]),
        "se_cash_ratio_cluster": float(fit.bse["cash_ratio"]),
        "se_log_sales_cluster": float(fit.bse["log_sales"]),
        "ci95_leverage_cluster": [float(v) for v in fit.conf_int().loc["leverage"]],
        "p_leverage_cluster": float(fit.pvalues["leverage"]),
        "se_leverage_hc1": float(fit_hc.bse["leverage"]),
        "n_clusters": int(len(np.unique(groups))),
        "cluster_se_note": "statsmodels cluster-robust covariance with default small-sample correction (G/(G-1))*((N-1)/(N-K)); CI uses normal critical values.",
    },
}
with open(f"{OUT}/results.json", "w") as f:
    json.dump(res, f, indent=2)
print(f"n={res['n_analysed']} firms={res['n_firms']} beta_lev={res['beta_leverage']:.4f} se={res['se_leverage_cluster']:.4f} r2={res['r2']:.4f} rate={res['default_rate']:.4f}")