"""F1-conf-03: information-respecting LPM of next-year default on leverage.
Run from workspace root: ~/wp29/venv/bin/python out/analysis.py
Reads only data/firm_panel.csv; writes out/results.json, out/analytic.csv, out/exclusions.csv."""
import json, os
import numpy as np
import pandas as pd
import statsmodels.api as sm

IN = "data/firm_panel.csv"
OUT = "out"
os.makedirs(OUT, exist_ok=True)
PRED = ["leverage", "cash_ratio", "log_sales"]

raw = pd.read_csv(IN, dtype={"firm_id": str})
n_raw = len(raw)
ex = []
def step(name, rule, before, after, why):
    ex.append(dict(step=name, rule=rule, n_before=before, n_after=after,
                   n_removed=before - after, rationale=why))

issues = []
# Step 1: column exclusion (information rule)
d = raw.drop(columns=["rating_2025"])
step("1_drop_column_rating_2025", "drop column rating_2025 (known only in 2025)", n_raw, len(d),
     "Final-vintage field assigned in 2025 using realised default history through 2022 and sample-average leverage; not known at end of year t (look-ahead leakage).")
issues.append(dict(issue="rating_2025 is a final-vintage (2025) field built from realised defaults through 2022 and average sample leverage: it violates the information-availability rule.",
                   handling="Column dropped from the analytic dataset; listed in variables_excluded; never used."))

# Step 2: exact duplicates
n_full_dup = int(raw.duplicated().sum())
n_key_dup = int(raw.duplicated(["firm_id", "year"]).sum())
b = len(d)
d = d[~raw.duplicated()].copy()
step("2_drop_exact_duplicates", "drop rows identical to an earlier row on all 7 columns", b, len(d),
     "Assembly error: firm-year rows repeated verbatim; keeping them would double-count observations.")
issues.append(dict(issue=f"{n_full_dup} exact duplicate rows (same firm_id-year, all fields identical); input has {n_raw} rows vs 2100 intended (300 firms x 7 years).",
                   handling="Kept first occurrence, dropped the copies (ledger step 2)."))
assert not d.duplicated(["firm_id", "year"]).any(), "conflicting firm-year duplicates remain"

# Step 3: unobserved outcome
b = len(d)
miss_years = sorted(d.loc[d.default_next.isna(), "year"].unique().tolist())
d = d[d.default_next.notna()].copy()
step("3_drop_missing_outcome", "drop firm-years with default_next empty", b, len(d),
     f"Outcome year t+1 not observed by the data cut (end 2022); occurs in years {miss_years}. Outcomes not imputed.")
issues.append(dict(issue=f"default_next missing (right-censored at the end-2022 data cut) for origin years {miss_years}.",
                   handling="Firm-years excluded from estimation (ledger step 3); no imputation."))

# Step 4: missing predictors (none expected, recorded for completeness)
b = len(d)
d = d.dropna(subset=PRED)
step("4_drop_missing_predictors", "drop rows with missing leverage/cash_ratio/log_sales", b, len(d),
     "Complete-case requirement for OLS (no rows affected if 0 removed).")

n_floor = int((d.leverage == d.leverage.min()).sum())
issues.append(dict(issue=f"leverage has a mass point at its minimum value {d.leverage.min()} ({n_floor} analysed firm-years), suggesting a floor/clipping in the source.",
                   handling="Kept as recorded; noted as a limitation (no correction applied)."))

d["default_next"] = d["default_next"].astype(int)
assert set(d.default_next.unique()) <= {0, 1}
d = d.sort_values(["firm_id", "year"]).reset_index(drop=True)
analytic = d[["firm_id", "year"] + PRED + ["default_next"]]
analytic.to_csv(f"{OUT}/analytic.csv", index=False)
pd.DataFrame(ex).to_csv(f"{OUT}/exclusions.csv", index=False)

X = sm.add_constant(analytic[PRED])
groups = pd.factorize(analytic.firm_id)[0]
fit = sm.OLS(analytic.default_next, X).fit(cov_type="cluster", cov_kwds={"groups": groups})
fit_hc = sm.OLS(analytic.default_next, X).fit(cov_type="HC1")

res = {
    "task_id": "F1-conf-03",
    "n_analysed": int(len(analytic)),
    "n_firms": int(analytic.firm_id.nunique()),
    "predictors_used": PRED,
    "variables_excluded": [
        {"variable": "rating_2025", "reason": "Known only in 2025; built from realised default history through 2022 and sample-average leverage (look-ahead leakage); violates information-availability rule."},
        {"variable": "default_next", "reason": "Outcome (dependent variable), known only at end of t+1; not a predictor."},
        {"variable": "firm_id", "reason": "Identifier; used only for clustering standard errors, not as a regressor."},
        {"variable": "year", "reason": "Origin-year identifier; not part of the specified model."},
    ],
    "beta_leverage": float(fit.params["leverage"]),
    "se_leverage_cluster": float(fit.bse["leverage"]),
    "r2": float(fit.rsquared),
    "default_rate": float(analytic.default_next.mean()),
    "feasibility": "complete",
    "feasibility_reason": None,
    "data_issues": issues,
    "n_input_rows": {"firm_panel.csv": int(n_raw)},
    "random_seed": None,
    # supplementary (not required)
    "supp_ci95_leverage_cluster": [float(x) for x in fit.conf_int().loc["leverage"]],
    "supp_pvalue_leverage_cluster": float(fit.pvalues["leverage"]),
    "supp_se_leverage_hc1": float(fit_hc.bse["leverage"]),
    "supp_coefficients": {k: float(v) for k, v in fit.params.items()},
    "supp_se_cluster": {k: float(v) for k, v in fit.bse.items()},
    "supp_n_clusters": int(len(np.unique(groups))),
    "supp_years_analysed": [int(analytic.year.min()), int(analytic.year.max())],
    "supp_n_exact_duplicates": n_full_dup,
    "supp_n_key_duplicates": n_key_dup,
}
with open(f"{OUT}/results.json", "w") as f:
    json.dump(res, f, indent=2)
print(f"n_analysed={res['n_analysed']} n_firms={res['n_firms']} beta_leverage={res['beta_leverage']:.4f} "
      f"se_cluster={res['se_leverage_cluster']:.4f} r2={res['r2']:.4f} default_rate={res['default_rate']:.4f}")