"""F1-conf-03 analysis: builds the information-respecting analytic dataset and fits the clustered LPM.
Run from workspace root: ~/wp29/venv/bin/python out/analysis.py"""
import json, os
import numpy as np, pandas as pd
import statsmodels.api as sm

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
OUT = os.path.join(ROOT, "out")
raw = pd.read_csv(os.path.join(ROOT, "data", "firm_panel.csv"), dtype={"firm_id": str})
n_raw = len(raw)
PRED = ["leverage", "cash_ratio", "log_sales"]
ledger = []
def step(name, rule, before, after, why):
    ledger.append(dict(step=name, rule=rule, n_before=before, n_after=after, n_removed=before - after, rationale=why))

# 1. drop leaking column (column removal, no rows removed)
d = raw.drop(columns=["rating_2025"])
step("1_drop_rating_2025", "drop column rating_2025 (known 2025, built from realised defaults through 2022)",
     n_raw, len(d), "violates information-availability rule; column removal only")
# 2. exact duplicate rows
n_exact_dup = int(raw.duplicated().sum())
n_key_dup = int(raw.duplicated(["firm_id", "year"]).sum())
b = len(d); d = d.loc[~raw.duplicated()].copy()
step("2_dedup", "remove exact duplicate rows (identical firm_id, year and all values in the original file); keep first",
     b, len(d), "assembly error: same firm-year listed twice")
assert not d.duplicated(["firm_id", "year"]).any()
# 3. unobserved outcome
n_missing_y = int(d.default_next.isna().sum())
years_missing = sorted(int(y) for y in d.loc[d.default_next.isna(), "year"].unique())
b = len(d); d = d.dropna(subset=["default_next"]).copy()
step("3_outcome_observed", "drop firm-years with default_next empty", b, len(d),
     f"outcome year not observed at data cut (origin years {years_missing}); no imputation")
# 4. missing predictors (none expected)
b = len(d); d = d.dropna(subset=PRED).copy()
step("4_complete_predictors", "drop rows with missing leverage/cash_ratio/log_sales", b, len(d), "complete-case for regressors")

d["default_next"] = d["default_next"].astype(int)
d = d.sort_values(["firm_id", "year"]).reset_index(drop=True)
d[["firm_id", "year"] + PRED + ["default_next"]].to_csv(os.path.join(OUT, "analytic.csv"), index=False)
pd.DataFrame(ledger).to_csv(os.path.join(OUT, "exclusions.csv"), index=False)

# Baseline: unconditional default rate
default_rate = float(d.default_next.mean())
# Model
X = sm.add_constant(d[PRED]); y = d.default_next
codes = pd.factorize(d.firm_id)[0]
fit = sm.OLS(y, X).fit(cov_type="cluster", cov_kwds={"groups": codes})
fit_naive = sm.OLS(y, X).fit()

res = {
    "task_id": "F1-conf-03",
    "n_analysed": int(len(d)),
    "n_firms": int(d.firm_id.nunique()),
    "predictors_used": PRED,
    "variables_excluded": [
        {"variable": "rating_2025", "reason": "Final-vintage field assigned in 2025 using realised default history through 2022 and sample-average leverage; not known at end of origin year t (outcome leakage)."},
    ],
    "beta_leverage": float(fit.params["leverage"]),
    "se_leverage_cluster": float(fit.bse["leverage"]),
    "r2": float(fit.rsquared),
    "default_rate": default_rate,
    "feasibility": "complete",
    "feasibility_reason": None,
    "data_issues": [
        {"issue": f"{n_exact_dup} exact duplicate rows ({n_key_dup} duplicated firm_id-year keys, all with identical values)",
         "handling": "kept one copy of each; after dedup the panel is balanced (300 firms x 7 years)"},
        {"issue": f"{n_missing_y} unique firm-years with empty default_next (origin years {years_missing}; outcome year not observed by end-2022 data cut)",
         "handling": "excluded from estimation, not imputed; listed in exclusions.csv"},
        {"issue": "rating_2025 is a post-period (2025) field derived from realised defaults: information leakage",
         "handling": "excluded from predictors and from analytic.csv"},
    ],
    "n_input_rows": {"firm_panel.csv": int(n_raw)},
    "random_seed": None,
    # supplementary (not required)
    "supp_ci95_leverage_cluster": [float(x) for x in fit.conf_int().loc["leverage"]],
    "supp_pvalue_leverage_cluster": float(fit.pvalues["leverage"]),
    "supp_se_leverage_iid": float(fit_naive.bse["leverage"]),
    "supp_coef_cash_ratio": float(fit.params["cash_ratio"]),
    "supp_coef_log_sales": float(fit.params["log_sales"]),
    "supp_intercept": float(fit.params["const"]),
    "supp_n_defaults": int(d.default_next.sum()),
}
with open(os.path.join(OUT, "results.json"), "w") as f:
    json.dump(res, f, indent=2)
print(json.dumps({k: res[k] for k in ["n_analysed", "n_firms", "beta_leverage", "se_leverage_cluster", "r2", "default_rate"]}, indent=2))