"""F1-conf-03: regenerate results.json, analytic.csv, exclusions.csv from data/ only."""
import json, os
import numpy as np, pandas as pd
import statsmodels.formula.api as smf

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
OUT = os.path.join(ROOT, "out")
raw = pd.read_csv(os.path.join(ROOT, "data", "firm_panel.csv"))
n_raw = len(raw)
PRED = ["leverage", "cash_ratio", "log_sales"]
ledger, issues = [], []
def step(name, rule, before, after, why):
    ledger.append(dict(step=name, rule=rule, n_before=before, n_after=after,
                       n_removed=before - after, rationale=why))

step("0_read", "rows read from firm_panel.csv", n_raw, n_raw, "starting point")

# Step 1: drop leakage column (column removal, no rows)
d = raw.drop(columns=["rating_2025"])
step("1_drop_rating_2025", "exclude column rating_2025 (known 2025)", n_raw, n_raw,
     "final-vintage field built from realised defaults through 2022 and sample-average leverage; not known at end of year t (look-ahead leakage)")

# Step 2: exact duplicate rows
n_exact = int(d.duplicated().sum())
assert int(raw.duplicated().sum()) == n_exact
b = len(d); d = d.drop_duplicates().reset_index(drop=True)
step("2_drop_exact_duplicates", "remove exact duplicate firm-year rows", b, len(d),
     "assembly error: identical rows repeated; one copy kept")
assert not d.duplicated(["firm_id", "year"]).any(), "conflicting firm-year duplicates"
issues.append(dict(issue=f"{n_exact} exact duplicate rows (same firm_id, year and all values)",
                   handling="dropped; first copy kept, giving one row per firm-year (300 firms x 7 years)"))

# Step 3: outcome not observed
b = len(d); miss = d["default_next"].isna()
miss_years = sorted(d.loc[miss, "year"].unique().tolist())
d = d[~miss].copy()
step("3_drop_unobserved_outcome", "remove firm-years with default_next empty", b, len(d),
     f"outcome year not observed by data cut (origin years {miss_years}); outcomes not imputed")
issues.append(dict(issue=f"default_next missing for {int(miss.sum())} unique firm-years (origin years {miss_years}: outcome year 2023 after the end-2022 data cut)",
                   handling="excluded from estimation, not imputed"))

# Step 4: missing predictors
b = len(d); d = d.dropna(subset=PRED)
step("4_drop_missing_predictors", "remove rows with missing leverage/cash_ratio/log_sales", b, len(d),
     "complete-case requirement for OLS (none found)")

n_floor = int((raw.drop_duplicates()["leverage"] == 0.02).sum())
issues.append(dict(issue=f"leverage takes the value 0.02 exactly in {n_floor} unique firm-years (minimum of the variable), suggesting bottom-coding/censoring at 0.02",
                   handling="kept as recorded; noted as limitation (cannot recover true values)"))
issues.append(dict(issue="rating_2025 is a final-vintage (2025) field constructed from realised default history and sample-average leverage; it is perfectly informative of default here (all defaults occur in rating D)",
                   handling="excluded from the analytic dataset as look-ahead leakage"))

d["default_next"] = d["default_next"].astype(int)
d = d.sort_values(["firm_id", "year"]).reset_index(drop=True)
d[["firm_id", "year"] + PRED + ["default_next"]].to_csv(os.path.join(OUT, "analytic.csv"), index=False)
pd.DataFrame(ledger).to_csv(os.path.join(OUT, "exclusions.csv"), index=False)

codes = pd.factorize(d["firm_id"])[0]
m = smf.ols("default_next ~ leverage + cash_ratio + log_sales", data=d).fit(
    cov_type="cluster", cov_kwds={"groups": codes})
res = dict(
    task_id="F1-conf-03",
    n_analysed=int(len(d)),
    n_firms=int(d["firm_id"].nunique()),
    predictors_used=PRED,
    variables_excluded=[dict(variable="rating_2025",
        reason="becomes known in 2025; built from realised default history through 2022 and average leverage over the sample -> violates end-of-year-t information rule (outcome leakage)"),
        dict(variable="default_next", reason="outcome variable (known end of year t+1); used only as dependent variable"),
        dict(variable="firm_id", reason="identifier; used only for clustering"),
        dict(variable="year", reason="origin-year index; not a regressor in the specified model")],
    beta_leverage=float(m.params["leverage"]),
    se_leverage_cluster=float(m.bse["leverage"]),
    r2=float(m.rsquared),
    default_rate=float(d["default_next"].mean()),
    feasibility="complete",
    feasibility_reason=None,
    data_issues=issues,
    n_input_rows={"firm_panel.csv": int(n_raw)},
    random_seed=None,
    extra=dict(ci95_leverage=[float(x) for x in m.conf_int().loc["leverage"]],
               p_leverage=float(m.pvalues["leverage"]),
               params={k: float(v) for k, v in m.params.items()},
               se_cluster={k: float(v) for k, v in m.bse.items()},
               se_cluster_note="statsmodels cluster-robust (CR1 with small-sample correction), 300 clusters"),
)
with open(os.path.join(OUT, "results.json"), "w") as f:
    json.dump(res, f, indent=2)
print(json.dumps(res, indent=2))