"""F1-conf-03: information-respecting analytic dataset and LPM for next-year default.
Run from workspace root: ~/wp29/venv/bin/python out/analysis.py"""
import json, os
import numpy as np, pandas as pd
import statsmodels.api as sm

IN = "data/firm_panel.csv"; OUT = "out"
os.makedirs(OUT, exist_ok=True)
raw = pd.read_csv(IN, dtype={"firm_id": str})
n_read = len(raw)
PRED = ["leverage", "cash_ratio", "log_sales"]
ledger, issues = [], []

def step(name, rule, before, after, why):
    ledger.append(dict(step=name, rule=rule, n_before=before, n_after=after,
                       n_removed=before - after, rationale=why))

step("0_read", "rows read from firm_panel.csv", n_read, n_read, "all data rows (excluding header)")

# column exclusion: rating_2025 is known only in 2025 and built from realised defaults
step("1_drop_rating_2025", "drop column rating_2025 (known 2025 > end of year t)", n_read, n_read,
     "final-vintage field built from realised default history through 2022 and sample-average leverage; "
     "look-ahead / outcome leakage; column removal only")

# exact duplicate rows
n_dup_exact = int(raw.duplicated().sum())
d = raw.drop_duplicates().copy()
step("2_drop_exact_duplicates", "remove exact duplicate rows (keep first)", n_read, len(d),
     "assembly error: identical firm-year rows repeated")
if n_dup_exact:
    issues.append(dict(issue=f"{n_dup_exact} exact duplicate rows (same firm_id-year, all fields identical)",
                       handling="removed, keeping one copy per firm-year"))
n_key_dup = int(d.duplicated(["firm_id", "year"]).sum())
if n_key_dup:  # conflicting duplicates (none expected after exact dedup check)
    issues.append(dict(issue=f"{n_key_dup} conflicting firm-year duplicates",
                       handling="all copies of conflicting firm-years dropped"))
    bad = d.duplicated(["firm_id", "year"], keep=False)
    b = len(d); d = d[~bad]
    step("2b_drop_conflicting_duplicates", "drop firm-years with conflicting rows", b, len(d), "cannot tell which row is correct")

# missing outcome
b = len(d)
miss = d["default_next"].isna()
miss_years = sorted(d.loc[miss, "year"].unique().tolist())
d = d[~miss].copy()
step("3_drop_missing_outcome", "drop firm-years with default_next not observed", b, len(d),
     f"outcome year t+1 not observed at data cut (end 2022); missing years: {miss_years}; not imputed")
issues.append(dict(issue=f"{int(miss.sum())} firm-years with missing default_next (origin years {miss_years}; "
                   "outcome year 2023 beyond data cut)", handling="excluded from estimation; not imputed"))

# missing predictors
b = len(d)
d = d.dropna(subset=PRED)
step("4_drop_missing_predictors", "drop rows with missing leverage/cash_ratio/log_sales", b, len(d),
     "complete-case on predictors")
if b != len(d):
    issues.append(dict(issue=f"{b-len(d)} rows with missing predictors", handling="dropped (complete case)"))

issues.append(dict(issue="rating_2025 is a final-vintage (2025) field derived from realised defaults through 2022 and average sample leverage",
                   handling="excluded from analytic dataset and model (information rule)"))
nd = d.groupby("firm_id")["default_next"].sum()
issues.append(dict(issue=f"default is not absorbing: {int((nd > 1).sum())} firms have default_next=1 in more than one year",
                   handling="kept as given (default treated as a repeatable annual event); noted as limitation"))

d["default_next"] = d["default_next"].astype(int)
d = d.sort_values(["firm_id", "year"]).reset_index(drop=True)
d[["firm_id", "year"] + PRED + ["default_next"]].to_csv(f"{OUT}/analytic.csv", index=False)
pd.DataFrame(ledger).to_csv(f"{OUT}/exclusions.csv", index=False)

X = sm.add_constant(d[PRED]); y = d["default_next"]
groups = pd.factorize(d["firm_id"])[0]
fit = sm.OLS(y, X).fit(cov_type="cluster", cov_kwds={"groups": groups})
res = dict(
    task_id="F1-conf-03",
    n_analysed=int(len(d)), n_firms=int(d["firm_id"].nunique()),
    predictors_used=PRED,
    variables_excluded=[
        dict(variable="rating_2025", reason="known only in 2025; built from realised default history through 2022 and "
             "sample-average leverage, so it leaks future outcomes (violates end-of-year-t availability)"),
        dict(variable="default_next", reason="outcome variable (known end of t+1); used as dependent variable, not predictor"),
        dict(variable="firm_id", reason="identifier; used only for clustering"),
        dict(variable="year", reason="origin-year index; not in specified model"),
    ],
    beta_leverage=float(fit.params["leverage"]),
    se_leverage_cluster=float(fit.bse["leverage"]),
    r2=float(fit.rsquared),
    default_rate=float(y.mean()),
    feasibility="complete", feasibility_reason=None,
    data_issues=issues,
    n_input_rows={"firm_panel.csv": int(n_read)},
    random_seed=None,
    _extra=dict(coef={k: float(v) for k, v in fit.params.items()},
                se_cluster={k: float(v) for k, v in fit.bse.items()},
                ci95_leverage=[float(x) for x in fit.conf_int().loc["leverage"]],
                p_leverage=float(fit.pvalues["leverage"]),
                cov_type="cluster by firm_id (statsmodels default small-sample correction)"),
)
with open(f"{OUT}/results.json", "w") as f:
    json.dump(res, f, indent=2)
print(json.dumps(res, indent=2))