"""F1-conf-03: information-respecting analytic dataset and LPM for next-year default.
Run from workspace root: ~/wp29/venv/bin/python out/analysis.py"""
import json, os
import pandas as pd
import statsmodels.formula.api as smf

os.makedirs("out", exist_ok=True)
raw = pd.read_csv("data/firm_panel.csv", dtype={"firm_id": str})
n_read = len(raw)
PRED = ["leverage", "cash_ratio", "log_sales"]
ledger, issues = [], []

def step(rule, before, after, why):
    ledger.append(dict(step=len(ledger) + 1, rule=rule, n_before=before, n_after=after,
                       n_removed=before - after, rationale=why))

step("rows read", n_read, n_read, "data rows in data/firm_panel.csv (header excluded)")

# 1. column removal: post-origin (look-ahead) variable
step("drop column rating_2025", n_read, n_read,
     "rating_2025 is known only in 2025 and was built from realised defaults through 2022 and "
     "full-sample average leverage: look-ahead/outcome leakage; column removed, no rows removed")
d = raw.drop(columns=["rating_2025"])

# 2. exact duplicate rows
n_dup = int(raw.duplicated().sum())
n_key_dup = int(raw.duplicated(["firm_id", "year"]).sum())
d = d.loc[~raw.duplicated()].copy()
assert not d.duplicated(["firm_id", "year"]).any()
step("drop exact duplicate rows", n_read, len(d),
     "identical copies of the same firm-year (assembly error); first copy kept")
issues.append(dict(issue=f"{n_dup} exact duplicate rows ({n_key_dup} duplicated firm_id-year keys; all "
                         "duplicates identical in every column), giving 40 firms 8 rows instead of 7",
                   handling="dropped duplicate copies, keeping one row per firm-year"))

# 3. missing outcome
before = len(d)
miss = d["default_next"].isna()
miss_years = sorted(d.loc[miss, "year"].unique().tolist())
d = d.loc[~miss].copy()
step("drop rows with missing default_next", before, len(d),
     f"outcome not observed (years {miss_years}: t+1 after the 2022 data cut); not imputed")
issues.append(dict(issue=f"{int(miss.sum())} firm-years with missing default_next, all in year(s) {miss_years} "
                         "(outcome year 2023 not observed at end-2022 cut)",
                   handling="excluded from estimation and recorded in the exclusion ledger; no imputation"))

# 4. missing predictors (none expected)
before = len(d)
d = d.dropna(subset=PRED)
step("drop rows with missing predictors", before, len(d), "complete-case on leverage, cash_ratio, log_sales")

issues.append(dict(issue="rating_2025 is a final-vintage (2025) field derived from realised defaults and average "
                         "leverage; it perfectly separates defaulters (all defaults have rating D)",
                   handling="excluded from the analytic dataset and the model (information rule)"))
n_floor = int((d["leverage"] == d["leverage"].min()).sum())
issues.append(dict(issue=f"leverage has a mass point at its minimum {d['leverage'].min()} ({n_floor} analysed rows), "
                         "suggesting floor-coding/censoring in the extract",
                   handling="kept as recorded; noted as a limitation"))
nd = d.groupby("firm_id")["default_next"].sum()
issues.append(dict(issue=f"{int((nd > 1).sum())} firms have default_next = 1 in more than one year (no exit after default)",
                   handling="kept; each firm-year treated as an observation as the estimand specifies; firm-clustered SEs"))

d["default_next"] = d["default_next"].astype(int)
d = d.sort_values(["firm_id", "year"]).reset_index(drop=True)
d[["firm_id", "year"] + PRED + ["default_next"]].to_csv("out/analytic.csv", index=False)
pd.DataFrame(ledger).to_csv("out/exclusions.csv", index=False)

m = smf.ols("default_next ~ leverage + cash_ratio + log_sales", data=d).fit(
    cov_type="cluster", cov_kwds={"groups": pd.factorize(d["firm_id"])[0]})

res = dict(
    task_id="F1-conf-03",
    n_analysed=int(len(d)),
    n_firms=int(d["firm_id"].nunique()),
    predictors_used=PRED,
    variables_excluded=[dict(variable="rating_2025",
                             reason="known only in 2025 (after origin year t); constructed from realised default "
                                    "history through 2022 and full-sample average leverage, so it leaks the outcome")],
    beta_leverage=float(m.params["leverage"]),
    se_leverage_cluster=float(m.bse["leverage"]),
    r2=float(m.rsquared),
    default_rate=float(d["default_next"].mean()),
    feasibility="complete",
    feasibility_reason=None,
    data_issues=issues,
    n_input_rows={"firm_panel.csv": int(n_read)},
    random_seed=None,
    # supplementary
    ci95_leverage=[float(x) for x in m.conf_int().loc["leverage"]],
    p_leverage=float(m.pvalues["leverage"]),
    coef_all={k: float(v) for k, v in m.params.items()},
    se_cluster_all={k: float(v) for k, v in m.bse.items()},
)
with open("out/results.json", "w") as f:
    json.dump(res, f, indent=2)
print(json.dumps(res, indent=2))