"""Replication entry point for F6-conf-02. Run from workspace root:
    ~/wp29/venv/bin/python out/analysis.py
Reads only data/gc5y_daily.csv; writes out/results.json, out/table1.csv,
out/figure1.png, out/exclusions.csv. No randomness is used."""
import json, os
import numpy as np
import pandas as pd
from scipy import stats
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt

TASK_ID = "F6-conf-02"
FN = "gc5y_daily.csv"
PRE_N = 120
os.makedirs("out", exist_ok=True)

raw = pd.read_csv(os.path.join("data", FN))
n_raw = len(raw)
df = raw.copy()
df["date"] = pd.to_datetime(df["date"], format="%Y-%m-%d")
df = df.sort_values("date").reset_index(drop=True)

# ---- audit ----
issues = []
n_missing = int(df.isna().sum().sum())
n_dup = int(df["date"].duplicated().sum())
if n_missing:
    issues.append({"issue": f"{n_missing} missing values", "handling": "rows with missing yield dropped"})
if n_dup:
    issues.append({"issue": f"{n_dup} duplicate dates", "handling": "reported; not removed"})
n_treated = int((df["post_cut"] == 1).sum())
sample_start = df["date"].min().strftime("%Y-%m-%d")
sample_end = df["date"].max().strftime("%Y-%m-%d")
if n_treated == 0:
    issues.append({
        "issue": f"No rows with post_cut = 1: the file covers {sample_start} to {sample_end} "
                 "(README retrieval used end_date=2019-12-31), entirely before the 2020-03-04 cut; "
                 "post_cut is 0 for all rows.",
        "handling": "Estimand not computable (no treated window, and the 120-day pre-window is defined "
                    "relative to the first flagged day, which does not exist). post_cut not redefined; "
                    "no proxy estimate substituted; effect keys set to null, conclusion premise_invalid."})
mismatch = int(((df["date"] >= "2020-03-04") != (df["post_cut"] == 1)).sum())
if mismatch:
    issues.append({"issue": f"{mismatch} rows where post_cut disagrees with its documented definition",
                   "handling": "post_cut used exactly as provided, per task rules"})

# ---- changes and baseline ----
df["chg_bp"] = 100.0 * df["yield_5y"].diff()
chg = df["chg_bp"].dropna()
baseline_metric = float(chg.mean())
excl = [{"step": 1, "rule": "first row has no previous row: one-day change undefined",
         "n_before": n_raw, "n_after": len(chg), "n_removed": n_raw - len(chg),
         "rationale": "changes are between consecutive rows (TASK.md)"}]

# ---- main estimate ----
effect = lo = hi = rmse = None
n_analysed = None
feas, feas_reason, conclusion = "complete", None, None
if n_treated > 0:
    first = df.index[df["post_cut"] == 1][0]
    post = df.loc[df["post_cut"] == 1, "chg_bp"].dropna()
    pre = df.loc[max(first - PRE_N, 0):first - 1, "chg_bp"].dropna()
    if len(pre) >= 2 and len(post) >= 2:
        effect = float(post.mean() - pre.mean())
        v1, v0 = post.var(ddof=1) / len(post), pre.var(ddof=1) / len(pre)
        se = np.sqrt(v1 + v0)
        dfw = (v1 + v0) ** 2 / (v1 ** 2 / (len(post) - 1) + v0 ** 2 / (len(pre) - 1))
        t = stats.t.ppf(0.975, dfw)
        lo, hi = float(effect - t * se), float(effect + t * se)
        resid = np.concatenate([post - post.mean(), pre - pre.mean()])
        rmse = float(np.sqrt(np.mean(resid ** 2)))
        n_analysed = int(len(post) + len(pre))
        conclusion = "positive_effect" if lo > 0 else "negative_effect" if hi < 0 else "no_detectable_effect"
        excl.append({"step": 2, "rule": "keep 120 pre rows + post_cut=1 rows",
                     "n_before": len(chg), "n_after": n_analysed, "n_removed": len(chg) - n_analysed,
                     "rationale": "estimand windows"})
if conclusion is None:
    feas, conclusion = "infeasible", "premise_invalid"
    feas_reason = (f"The file contains no rows with post_cut = 1 (sample {sample_start} to {sample_end}, "
                   "ending before the 2020-03-04 cut), so the post-cut mean change and the 120-day "
                   "pre-window anchored at the first flagged day cannot be computed.")
    excl.append({"step": 2, "rule": "keep 120 rows before first post_cut=1 row + post_cut=1 rows",
                 "n_before": len(chg), "n_after": 0, "n_removed": len(chg),
                 "rationale": "no post_cut=1 rows exist, so both estimand windows are empty"})
pd.DataFrame(excl).to_csv("out/exclusions.csv", index=False)

# ---- table and figure ----
df["year"] = df["date"].dt.year
tab = df.groupby("year").agg(n_days=("yield_5y", "size"), mean_yield=("yield_5y", "mean"),
                             mean_change_bp=("chg_bp", "mean")).reset_index()
tab.round(6).to_csv("out/table1.csv", index=False)

fig, ax = plt.subplots(figsize=(9, 4))
ax.plot(df["date"], df["yield_5y"], lw=0.8, color="tab:blue")
ax.axvline(pd.Timestamp("2020-03-04"), color="red", ls="--", lw=1, label="2020-03-04 cut (outside sample)")
ax.set_xlim(df["date"].min(), pd.Timestamp("2020-06-30"))
ax.set_ylabel("5-year GoC yield (%)")
ax.set_title(f"GoC 5-year benchmark yield, {sample_start} to {sample_end} (no post_cut=1 rows)"
             if n_treated == 0 else "GoC 5-year benchmark yield")
ax.legend(loc="upper right")
fig.tight_layout()
fig.savefig("out/figure1.png", dpi=120)

res = {
    "task_id": TASK_ID, "sample_start": sample_start, "sample_end": sample_end,
    "n_treated": n_treated, "baseline_metric": baseline_metric, "model_metric": rmse,
    "effect_estimate": effect, "effect_ci_low": lo, "effect_ci_high": hi,
    "conclusion": conclusion, "feasibility": feas, "feasibility_reason": feas_reason,
    "data_issues": issues, "n_input_rows": {FN: n_raw}, "n_analysed": n_analysed,
    "random_seed": None,
    "baseline_n_changes": int(len(chg)),
}
with open("out/results.json", "w") as f:
    json.dump(res, f, indent=2)
print(f"rows={n_raw} treated={n_treated} baseline_mean_change_bp={baseline_metric:.6f} conclusion={conclusion}")