"""F1-conf-01: 12-month CPI inflation from two ALFRED vintages.
Run from workspace root: ~/wp29/venv/bin/python out/analysis.py
Reads only data/; writes out/results.json, out/analytic.csv, out/exclusions.csv. No randomness."""
import json, os
import numpy as np
import pandas as pd

D, O = "data", "out"
F_OLD, F_NEW, F_ASM = "cpi_vintage_2016-07-15.csv", "cpi_vintage_2026-10-28.csv", "cpi_assembled.csv"
V_OLD, V_NEW = "2016-07-15", "2026-10-28"
W0, W1 = pd.Timestamp("2001-01-01"), pd.Timestamp("2026-09-01")

old = pd.read_csv(os.path.join(D, F_OLD), parse_dates=["date"])
new = pd.read_csv(os.path.join(D, F_NEW), parse_dates=["date"])
asm = pd.read_csv(os.path.join(D, F_ASM), parse_dates=["date"])
n_input_rows = {F_OLD: len(old), F_NEW: len(new), F_ASM: len(asm)}
issues = []

for name, df in [(F_OLD, old), (F_NEW, new)]:
    assert not df.date.duplicated().any(), name
    full = pd.date_range(df.date.min(), df.date.max(), freq="MS")
    assert len(full) == len(df), f"gap in {name}"
    for d in df.loc[df.cpi.isna(), "date"]:
        issues.append({"issue": f"{name}: no index value reported for {d:%Y-%m} (empty cpi cell)",
                       "handling": "Month treated as unavailable in that vintage; the older vintage does not cover it, so P is missing. "
                                   "Not interpolated; g_t for this month (and for t+12 if inside the window) reported as missing."})

# Splice: most recent vintage in which the month is available (non-empty value).
o = old.dropna(subset=["cpi"]).assign(vintage_used=V_OLD).set_index("date")
n = new.dropna(subset=["cpi"]).assign(vintage_used=V_NEW).set_index("date")
lev = pd.concat([n, o[~o.index.isin(n.index)]]).sort_index()
ov = old.merge(new, on="date", suffixes=("_old", "_new")).dropna()
issues.append({"issue": f"Vintages overlap for {ov.date.min():%Y-%m}..{ov.date.max():%Y-%m} ({len(ov)} months) with differing levels "
                        f"(max abs diff {np.abs(ov.cpi_new-ov.cpi_old).max():.3f} index points; seasonal-factor/revision differences)",
               "handling": "2026-10-28 vintage used for all overlap months per the most-recent-vintage rule; growth for 2014-01..2014-12 "
                           "uses a 2026-vintage numerator and a 2016-vintage (2013) denominator, as the rule implies."})

# Diagnose the hand-assembled file against the authoritative splice
a = asm.set_index("date")
full_rng = pd.date_range("1947-01-01", "2026-09-01", freq="MS")
missing_in_asm = [d for d in full_rng if d not in a.index]
extra_in_asm = [d for d in a.index if d not in full_rng]
if missing_in_asm:
    issues.append({"issue": f"{F_ASM}: months absent: {', '.join(f'{d:%Y-%m}' for d in missing_in_asm)}",
                   "handling": "Assembled file not used for estimation; levels taken from the two vintage files."})
if extra_in_asm:
    issues.append({"issue": f"{F_ASM}: months beyond the sources' coverage: {', '.join(f'{d:%Y-%m}' for d in extra_in_asm)} "
                            "(values appear shifted one month later from 2016-07 onward)",
                   "handling": "Assembled file not used for estimation."})
cmp_ = a.join(lev[["cpi", "vintage_used"]], how="inner", rsuffix="_auth")
vint_wrong = int((cmp_.vintage != cmp_.vintage_used).sum())
val_wrong = int((~np.isclose(cmp_.cpi, cmp_.cpi_auth, equal_nan=True)).sum())
issues.append({"issue": f"{F_ASM}: of {len(cmp_)} months shared with the authoritative splice, {val_wrong} have a different level and "
                        f"{vint_wrong} a different vintage label (2014-01..2016-06 taken from the older vintage; 2016-07 dropped and later values mis-dated)",
               "handling": "Assembled file read for diagnostics only; all estimates use the vintage files directly."})

# Growth rates
lev = lev.reindex(pd.date_range(lev.index.min(), lev.index.max(), freq="MS"))
lag = lev.cpi.shift(12)
g = 100 * (lev.cpi / lag - 1)
win = pd.date_range(W0, W1, freq="MS")
tab = pd.DataFrame({"cpi": lev.cpi, "cpi_lag12": lag, "vintage_used": lev.vintage_used, "growth_12m": g}).reindex(win)
ok = tab.growth_12m.notna()
analytic = tab[ok].copy()
missing_months = [f"{d:%Y-%m}" for d in tab.index[~ok]]

n2 = int(tab.cpi.notna().sum())
excl = [
    dict(step=1, rule="Window 2001-01..2026-09 (all calendar months)", n_before=len(win), n_after=len(win), n_removed=0,
         rationale="Start from every month of the target window"),
    dict(step=2, rule="Drop months with no index level P_t in the most recent covering vintage",
         n_before=len(win), n_after=n2, n_removed=len(win) - n2,
         rationale="No imputation allowed: " + (", ".join(f"{d:%Y-%m}" for d in tab.index[tab.cpi.isna()]) or "none")),
    dict(step=3, rule="Drop months with no index level P_{t-12}", n_before=n2, n_after=int(ok.sum()),
         n_removed=n2 - int(ok.sum()),
         rationale="No imputation allowed: " + (", ".join(f"{d:%Y-%m}" for d in tab.index[tab.cpi.notna() & tab.cpi_lag12.isna()]) or "none")),
]
pd.DataFrame(excl).to_csv(os.path.join(O, "exclusions.csv"), index=False)

out_a = analytic.reset_index().rename(columns={"index": "date"})
out_a["date"] = out_a.date.dt.strftime("%Y-%m-%d")
out_a.to_csv(os.path.join(O, "analytic.csv"), index=False, float_format="%.10g")

gs = analytic.growth_12m
def gm(m):
    v = tab.growth_12m.get(pd.Timestamp(m + "-01"))
    return None if v is None or pd.isna(v) else round(float(v), 10)
r = lambda x: round(float(x), 10)
last_month = lev.cpi.loc[:W1].dropna().index.max()
res = {
    "task_id": "F1-conf-01",
    "first_growth_month": f"{gs.index.min():%Y-%m}",
    "last_month": f"{last_month:%Y-%m}",
    "n_months_growth": int(ok.sum()),
    "n_months_missing_growth": int((~ok).sum()),
    "n_analysed": int(ok.sum()),
    "mean_growth_12m": r(gs.mean()),
    "mean_growth_12m_2001_2013": r(gs.loc[:"2013-12-01"].mean()),
    "mean_growth_12m_2014_2026": r(gs.loc["2014-01-01":].mean()),
    "growth_2014_06": gm("2014-06"), "growth_2016_07": gm("2016-07"), "growth_2020_05": gm("2020-05"),
    "growth_2022_06": gm("2022-06"), "growth_2026_09": gm("2026-09"),
    "max_growth_12m": r(gs.max()), "max_growth_month": f"{gs.idxmax():%Y-%m}",
    "min_growth_12m": r(gs.min()), "min_growth_month": f"{gs.idxmin():%Y-%m}",
    "feasibility": "complete" if ok.all() else "partial",
    "feasibility_reason": None if ok.all() else
        f"g_t not computable for {len(missing_months)} month(s) in the window ({', '.join(missing_months)}) because the source reports no index level; "
        "these months are excluded from all means and extremes. All summary estimands were computed.",
    "data_issues": issues,
    "n_input_rows": n_input_rows,
    "random_seed": None,
    "n_months_2001_2013": int(gs.loc[:"2013-12-01"].size),
    "n_months_2014_2026": int(gs.loc["2014-01-01":].size),
    "missing_growth_months": missing_months,
}
with open(os.path.join(O, "results.json"), "w") as f:
    json.dump(res, f, indent=2)
print(json.dumps({k: v for k, v in res.items() if k != "data_issues"}, indent=2))