"""F1-conf-03: information-respecting analytic dataset and LPM for next-year default.
Run from workspace root: ~/wp29/venv/bin/python out/analysis.py"""
import json, os
import numpy as np, pandas as pd
import statsmodels.formula.api as smf

IN = "data/firm_panel.csv"; OUT = "out"
os.makedirs(OUT, exist_ok=True)
raw = pd.read_csv(IN, dtype={"firm_id": str})
n_read = len(raw)
PRED = ["leverage", "cash_ratio", "log_sales"]
ledger, issues = [], []
def step(rule, before, after, why):
    ledger.append(dict(step=len(ledger) + 1, rule=rule, n_before=before, n_after=after,
                       n_removed=before - after, rationale=why))

# Step 1: rows read
step("rows read from firm_panel.csv", n_read, n_read, "starting point")

# Step 2: exact duplicate rows (assembly error)
n_exact = int(raw.duplicated().sum())
n_key = int(raw.duplicated(["firm_id", "year"]).sum())
d = raw.drop_duplicates().copy()
assert not d.duplicated(["firm_id", "year"]).any(), "conflicting firm-year duplicates"
step("drop exact duplicate rows (same firm_id-year, identical values)", n_read, len(d),
     "assembly error: rows repeated in extract; one copy kept")
issues.append(dict(issue=f"{n_exact} exact duplicate rows ({n_key} duplicated firm_id-year keys; all copies identical in every column)",
                   handling="dropped the redundant copies, kept one row per firm-year (300 firms x 7 years = 2100 rows)"))

# Step 3: column removal for look-ahead variable
step("exclude column rating_2025 (known only in 2025)", len(d), len(d),
     "final-vintage field built from realised defaults through 2022 and sample-average leverage: look-ahead / outcome leakage")
d = d.drop(columns=["rating_2025"])
issues.append(dict(issue="rating_2025 is a 2025 final-vintage field constructed from the realised default history (only class D has any defaults in the data)",
                   handling="excluded from predictors and from analytic.csv"))

# Step 4: unobserved outcome
before = len(d)
miss_years = sorted(d.loc[d.default_next.isna(), "year"].unique().tolist())
d = d[d.default_next.notna()].copy()
step("drop firm-years with unobserved default_next", before, len(d),
     f"outcome year t+1 not observed at data cut (end 2022); missing outcomes all in year(s) {miss_years}; not imputed")
issues.append(dict(issue=f"default_next missing for {before-len(d)} firm-years (all origin year {miss_years})",
                   handling="excluded from estimation sample (right-censored at the data cut), no imputation"))

# Step 5: missing predictors (none expected)
before = len(d)
d = d.dropna(subset=PRED)
step("drop firm-years with missing predictors", before, len(d), "complete-case requirement for OLS")

# Plausibility notes (no rows removed)
d["default_next"] = d["default_next"].astype(int)
s = d.sort_values(["firm_id", "year"])
rep = int(((s.groupby("firm_id").default_next.shift() == 1) & (s.default_next == 1)).sum())
issues.append(dict(issue=f"default is not absorbing: firms record default_next=1 in multiple years ({rep} firm-years with default following a prior-year default) and remain in the panel",
                   handling="kept as given (default treated as a recurring annual event); noted as limitation"))
issues.append(dict(issue=f"leverage bounded in [{d.leverage.min():.2f}, {d.leverage.max():.2f}], suggesting possible winsorisation/clipping in the simulation",
                   handling="kept as given"))

# Model
m = smf.ols("default_next ~ leverage + cash_ratio + log_sales", data=d).fit(
    cov_type="cluster", cov_kwds={"groups": pd.factorize(d["firm_id"])[0]})
ana = d[["firm_id", "year"] + PRED + ["default_next"]].sort_values(["firm_id", "year"])
ana.to_csv(f"{OUT}/analytic.csv", index=False)
pd.DataFrame(ledger).to_csv(f"{OUT}/exclusions.csv", index=False)

res = dict(
    task_id="F1-conf-03",
    n_analysed=int(len(ana)), n_firms=int(ana.firm_id.nunique()),
    predictors_used=PRED,
    variables_excluded=[dict(variable="rating_2025",
        reason="known only in 2025; built from realised default history through 2022 and average leverage over the sample (look-ahead / outcome leakage)")],
    beta_leverage=float(m.params["leverage"]),
    se_leverage_cluster=float(m.bse["leverage"]),
    r2=float(m.rsquared),
    default_rate=float(ana.default_next.mean()),
    feasibility="complete", feasibility_reason=None,
    data_issues=issues,
    n_input_rows={"firm_panel.csv": int(n_read)},
    random_seed=None,
    extra=dict(ci95_leverage=[float(x) for x in m.conf_int().loc["leverage"]],
               p_leverage=float(m.pvalues["leverage"]),
               coef=m.params.to_dict(), se_cluster=m.bse.to_dict(),
               n_clusters=int(ana.firm_id.nunique()),
               years=[int(ana.year.min()), int(ana.year.max())]),
)
with open(f"{OUT}/results.json", "w") as f:
    json.dump(res, f, indent=2)
print(json.dumps(res, indent=2)); print(m.summary())