"""F1-conf-01: 12-month CPI inflation from two ALFRED vintages.
Run from workspace root: ~/wp29/venv/bin/python out/analysis.py
Reads only data/; writes out/results.json, out/analytic.csv, out/exclusions.csv. No randomness."""
import json, os
import numpy as np
import pandas as pd

D, O = "data", "out"
F_OLD, F_NEW, F_ASM = "cpi_vintage_2016-07-15.csv", "cpi_vintage_2026-10-28.csv", "cpi_assembled.csv"
W0, W1 = pd.Timestamp("2001-01-01"), pd.Timestamp("2026-09-01")

old = pd.read_csv(os.path.join(D, F_OLD), parse_dates=["date"])
new = pd.read_csv(os.path.join(D, F_NEW), parse_dates=["date"])
asm = pd.read_csv(os.path.join(D, F_ASM), parse_dates=["date"])  # read for audit only
n_rows = {F_OLD: len(old), F_NEW: len(new), F_ASM: len(asm)}
issues = []

# --- sanity checks on authoritative vintages
for name, df in [(F_OLD, old), (F_NEW, new)]:
    assert not df.date.duplicated().any(), name
    assert (df.date.dt.day == 1).all(), name
    full = pd.date_range(df.date.min(), df.date.max(), freq="MS")
    assert len(full) == len(df), f"gap in date grid of {name}"

new_missing = new.loc[new.cpi.isna(), "date"].dt.strftime("%Y-%m").tolist()
if new_missing:
    issues.append({"issue": f"{F_NEW} has an empty cpi cell (no published value) for {', '.join(new_missing)}",
                   "handling": "Not imputed and not filled from the older vintage (the 2016 vintage does not cover it); "
                               "g_t for 2025-10 is missing and counted (the dependent month 2026-10 lies outside the window and outside source coverage)."})

# --- audit of the hand-assembled file against the vintages
a = asm.set_index("date")
exp_old = old.set_index("date").cpi
exp_new = new.set_index("date").cpi
overlap = exp_old.index.intersection(exp_new.index)
asm_ov = a.loc[a.index.intersection(overlap)]
n_old_in_overlap = int((asm_ov.vintage == "2016-07-15").sum())
if n_old_in_overlap:
    issues.append({"issue": f"{F_ASM}: {n_old_in_overlap} overlap months ({overlap.min():%Y-%m}..{overlap.max():%Y-%m}) take the 2016-07-15 value instead of the most recent (2026-10-28) vintage",
                   "handling": "Assembled file not used; series rebuilt from the two vintage files with the most-recent-vintage rule."})
missing_in_asm = [d for d in exp_new.index if d not in a.index]
if missing_in_asm:
    issues.append({"issue": f"{F_ASM}: months present in {F_NEW} are absent: {', '.join(d.strftime('%Y-%m') for d in missing_in_asm)}",
                   "handling": "Assembled file not used."})
asm_new = a[a.vintage == "2026-10-28"]
n_mis = int(sum(1 for d, r in asm_new.iterrows()
                if d not in exp_new.index or not ((pd.isna(r.cpi) and pd.isna(exp_new[d])) or np.isclose(r.cpi, exp_new[d]))))
if n_mis:
    beyond = [d.strftime('%Y-%m') for d in asm_new.index if d > new.date.max()]
    issues.append({"issue": f"{F_ASM}: {n_mis} rows labelled 2026-10-28 do not match {F_NEW} at the same date (values appear shifted by one month from 2016-08 onward; dates beyond source coverage: {beyond}; missing value placed at 2025-11 instead of 2025-10)",
                   "handling": "Assembled file not used."})

# --- build series: newest vintage where it covers the month, else old vintage
s_new = new.set_index("date")
s_old = old.set_index("date")
covered_new = s_new.index  # months covered by the 2026 vintage extract (incl. empty cell)
rows = []
for d in pd.date_range(old.date.min(), new.date.max(), freq="MS"):
    if d in covered_new:
        rows.append((d, s_new.cpi[d], "2026-10-28"))
    elif d in s_old.index:
        rows.append((d, s_old.cpi[d], "2016-07-15"))
    else:
        rows.append((d, np.nan, None))
ser = pd.DataFrame(rows, columns=["date", "cpi", "vintage_used"]).set_index("date")
assert ser.vintage_used.notna().all()
ser["cpi_lag12"] = ser.cpi.shift(12)  # contiguous monthly grid, so shift(12) = t-12
ser["growth_12m"] = 100.0 * (ser.cpi / ser.cpi_lag12 - 1.0)

win = ser.loc[W0:W1].copy()
n_win = len(win)
step1 = win[win.cpi.notna()]
step2 = step1[step1.cpi_lag12.notna()]
an = step2.copy()
excl = pd.DataFrame([
    dict(step=1, rule="window 2001-01..2026-09 drop months with no index level P_t in the sources",
         n_before=n_win, n_after=len(step1), n_removed=n_win - len(step1),
         rationale="no imputation of missing levels (empty cell in 2026-10-28 vintage)"),
    dict(step=2, rule="drop months whose lagged level P_{t-12} is missing",
         n_before=len(step1), n_after=len(step2), n_removed=len(step1) - len(step2),
         rationale="g_t not computable without P_{t-12}; no imputation"),
])
excl.to_csv(os.path.join(O, "exclusions.csv"), index=False)

out = an.reset_index()[["date", "cpi", "cpi_lag12", "vintage_used", "growth_12m"]]
out["date"] = out.date.dt.strftime("%Y-%m-01")
out.to_csv(os.path.join(O, "analytic.csv"), index=False, float_format="%.10g")

g = an.growth_12m
def gm(m):
    d = pd.Timestamp(m + "-01")
    return float(round(g[d], 10)) if d in g.index else None

missing_months = win.index[win.growth_12m.isna()].strftime("%Y-%m").tolist()
last_level = ser.loc[:W1].cpi.dropna().index.max()
res = {
    "task_id": "F1-conf-01",
    "first_growth_month": g.index.min().strftime("%Y-%m"),
    "last_month": last_level.strftime("%Y-%m"),
    "n_months_growth": int(len(g)),
    "n_months_missing_growth": int(n_win - len(g)),
    "n_analysed": int(len(g)),
    "mean_growth_12m": float(round(g.mean(), 10)),
    "mean_growth_12m_2001_2013": float(round(g.loc["2001-01-01":"2013-12-01"].mean(), 10)),
    "mean_growth_12m_2014_2026": float(round(g.loc["2014-01-01":"2026-09-01"].mean(), 10)),
    "growth_2014_06": gm("2014-06"), "growth_2016_07": gm("2016-07"), "growth_2020_05": gm("2020-05"),
    "growth_2022_06": gm("2022-06"), "growth_2026_09": gm("2026-09"),
    "max_growth_12m": float(round(g.max(), 10)), "max_growth_month": g.idxmax().strftime("%Y-%m"),
    "min_growth_12m": float(round(g.min(), 10)), "min_growth_month": g.idxmin().strftime("%Y-%m"),
    "feasibility": "complete" if not missing_months else "partial",
    "feasibility_reason": None if not missing_months else
        f"g_t not computable for {len(missing_months)} month(s) of the {n_win}-month window ({', '.join(missing_months)}) because the most recent vintage publishes no index level for 2025-10; all other estimands computed.",
    "data_issues": issues,
    "n_input_rows": n_rows,
    "random_seed": None,
    "n_window_months": int(n_win),
    "missing_growth_months": missing_months,
    "n_growth_2001_2013": int(len(g.loc["2001-01-01":"2013-12-01"])),
    "n_growth_2014_2026": int(len(g.loc["2014-01-01":"2026-09-01"])),
    "sd_growth_12m": float(round(g.std(ddof=1), 10)),
}
with open(os.path.join(O, "results.json"), "w") as f:
    json.dump(res, f, indent=2)
print(json.dumps({k: res[k] for k in ["n_analysed", "mean_growth_12m", "max_growth_month", "min_growth_month", "feasibility"]}))