"""F1-conf-01: 12-month CPI inflation from two ALFRED vintages.
Run from workspace root: ~/wp29/venv/bin/python out/analysis.py"""
import json
from pathlib import Path
import pandas as pd

D, O = Path("data"), Path("out")
O.mkdir(exist_ok=True)
F_OLD, F_NEW, F_ASM = "cpi_vintage_2016-07-15.csv", "cpi_vintage_2026-10-28.csv", "cpi_assembled.csv"
old = pd.read_csv(D / F_OLD, dtype={"date": str})
new = pd.read_csv(D / F_NEW, dtype={"date": str})
asm = pd.read_csv(D / F_ASM, dtype={"date": str, "vintage": str})
for df in (old, new, asm):
    df["date"] = pd.to_datetime(df["date"])
issues = []

# --- diagnose the hand-assembled file (used only for checking, not for estimation)
exp = pd.concat([old[old.date < "2014-01-01"].assign(vintage="2016-07-15"),
                 new.assign(vintage="2026-10-28")], ignore_index=True)
full = pd.DataFrame({"date": pd.date_range("1947-01-01", "2026-09-01", freq="MS")})
missing_dates = sorted(set(full.date) - set(asm.date))
if missing_dates:
    issues.append({"issue": f"cpi_assembled.csv has no row for {', '.join(d.strftime('%Y-%m') for d in missing_dates)} "
                            f"and contains a row dated {asm.date.max().strftime('%Y-%m')} (outside the vintage coverage); "
                            "from 2016-08 onward every value is shifted one month later than its true date "
                            "(e.g. the 2016-07 value 240.101 is labelled 2016-08).",
                   "handling": "cpi_assembled.csv not used for estimation; series rebuilt directly from the two authoritative vintage files."})
ov = asm[(asm.date >= "2014-01-01") & (asm.date <= "2016-06-01")]
if (ov.vintage == "2016-07-15").any():
    issues.append({"issue": f"cpi_assembled.csv uses the older 2016-07-15 vintage for {int((ov.vintage=='2016-07-15').sum())} overlap months "
                            "2014-01..2016-06, although the 2026-10-28 vintage covers them (values differ by seasonal-factor revisions).",
                   "handling": "Per the availability rule, the 2026-10-28 vintage value is used for every month it covers."})
nnew_na = new[new.cpi.isna()]
for d in nnew_na.date:
    issues.append({"issue": f"2026-10-28 vintage reports no value for {d.strftime('%Y-%m')} (empty cpi cell); "
                            "the 2016-07-15 vintage does not cover that month either.",
                   "handling": f"No interpolation; g_t for {d.strftime('%Y-%m')} (and for {(d+pd.DateOffset(months=12)).strftime('%Y-%m')}, outside the window) is not computable and is counted as missing."})
asm_na = asm[asm.cpi.isna()]
if len(asm_na):
    issues.append({"issue": f"cpi_assembled.csv has an empty cpi at row labelled {asm_na.date.iloc[0].strftime('%Y-%m')} (this is the 2025-10 gap, mislabelled by the one-month shift).",
                   "handling": "Not used; see shift issue."})
issues.append({"issue": "Growth rates for 2014-01..2014-12 combine a 2026-10-28-vintage numerator with a 2016-07-15-vintage lag (2013 months not in the recent extract); seasonal factors for 2013 were later revised.",
               "handling": "Applied the stated rule (most recent vintage available per month) as specified; flagged as a limitation in the report."})

# --- build the series per availability rule
new_av = new.dropna(subset=["cpi"])
ser = pd.concat([new_av.assign(vintage="2026-10-28"),
                 old[~old.date.isin(new.date)].assign(vintage="2016-07-15")])
ser = full.merge(ser, on="date", how="left")
s = ser.set_index("date")
lag = s.cpi.shift(12)  # index is complete monthly, so shift(12) = 12 months
lag_v = s.vintage.shift(12)
s["cpi_lag12"] = lag
s["growth_12m"] = 100 * (s.cpi / s.cpi_lag12 - 1)
s["vintage_used"] = s.vintage.where(lag_v == s.vintage, s.vintage + "/lag:" + lag_v.fillna(""))
win = s.loc["2001-01-01":"2026-09-01"].reset_index()
n_win = len(win)
ok = win.dropna(subset=["growth_12m"])
an = ok[["date", "cpi", "cpi_lag12", "vintage_used", "growth_12m"]].copy()
an["date"] = an.date.dt.strftime("%Y-%m-01")
an.to_csv(O / "analytic.csv", index=False, float_format="%.10g")

excl = pd.DataFrame([
    dict(step=1, rule="All calendar months in window 2001-01..2026-09", n_before=n_win, n_after=n_win, n_removed=0,
         rationale="Starting set: one row per month of the estimation window"),
    dict(step=2, rule="Drop months whose g_t cannot be computed (P_t or P_{t-12} absent in all vintages)",
         n_before=n_win, n_after=len(ok), n_removed=n_win - len(ok),
         rationale="No interpolation/imputation allowed; missing: " + ", ".join(win[win.growth_12m.isna()].date.dt.strftime("%Y-%m"))),
])
excl.to_csv(O / "exclusions.csv", index=False)

g = ok.set_index("date").growth_12m
def gm(m):
    v = s.growth_12m.get(pd.Timestamp(m + "-01"))
    return None if pd.isna(v) else round(float(v), 6)
last_level = ser.dropna(subset=["cpi"]).date.max().strftime("%Y-%m")
nmiss = n_win - len(ok)
res = {
    "task_id": "F1-conf-01",
    "first_growth_month": g.index.min().strftime("%Y-%m"),
    "last_month": last_level,
    "n_months_growth": int(len(ok)),
    "n_months_missing_growth": int(nmiss),
    "n_analysed": int(len(ok)),
    "mean_growth_12m": round(float(g.mean()), 6),
    "mean_growth_12m_2001_2013": round(float(g["2001-01-01":"2013-12-01"].mean()), 6),
    "mean_growth_12m_2014_2026": round(float(g["2014-01-01":"2026-09-01"].mean()), 6),
    "growth_2014_06": gm("2014-06"), "growth_2016_07": gm("2016-07"), "growth_2020_05": gm("2020-05"),
    "growth_2022_06": gm("2022-06"), "growth_2026_09": gm("2026-09"),
    "max_growth_12m": round(float(g.max()), 6), "max_growth_month": g.idxmax().strftime("%Y-%m"),
    "min_growth_12m": round(float(g.min()), 6), "min_growth_month": g.idxmin().strftime("%Y-%m"),
    "feasibility": "complete" if nmiss == 0 else "partial",
    "feasibility_reason": None if nmiss == 0 else
        f"{nmiss} month(s) of the window have no computable g_t because the source publishes no index level for 2025-10; "
        "summary statistics are over the remaining months.",
    "data_issues": issues,
    "n_input_rows": {F_OLD: int(len(old)), F_NEW: int(len(new)), F_ASM: int(len(asm))},
    "random_seed": None,
}
(O / "results.json").write_text(json.dumps(res, indent=2))
print(json.dumps(res, indent=2))

# --- sensitivity (not part of results.json): 2014 growth using the 2016-07-15 vintage for both P_t and P_{t-12}
o = old.set_index("date").cpi
g_old = (100 * (o / o.shift(12) - 1)).loc["2014-01-01":"2014-12-01"]
g_mix = s.growth_12m.loc["2014-01-01":"2014-12-01"]
sens = {"mean_2014_mixed_vintage": round(float(g_mix.mean()), 6),
        "mean_2014_old_vintage_only": round(float(g_old.mean()), 6),
        "max_abs_diff_2014_months": round(float((g_mix - g_old).abs().max()), 6),
        "growth_2014_06_old_vintage_only": round(float(g_old.loc["2014-06-01"]), 6),
        "mean_growth_12m_2001_2013_n": int(g["2001-01-01":"2013-12-01"].size),
        "mean_growth_12m_2014_2026_n": int(g["2014-01-01":"2026-09-01"].size),
        "sd_growth_12m": round(float(g.std()), 6)}
(O / "sensitivity.json").write_text(json.dumps(sens, indent=2))