1"""F1-conf-01: 12-month CPI inflation from two ALFRED vintages.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Reads only data/; writes out/results.json, out/analytic.csv, out/exclusions.csv. No randomness."""4import json, os5import numpy as np6import pandas as pd78D, O = "data", "out"9F_OLD, F_NEW, F_ASM = "cpi_vintage_2016-07-15.csv", "cpi_vintage_2026-10-28.csv", "cpi_assembled.csv"10V_OLD, V_NEW = "2016-07-15", "2026-10-28"11W0, W1 = pd.Timestamp("2001-01-01"), pd.Timestamp("2026-09-01")1213old = pd.read_csv(os.path.join(D, F_OLD), parse_dates=["date"])14new = pd.read_csv(os.path.join(D, F_NEW), parse_dates=["date"])15asm = pd.read_csv(os.path.join(D, F_ASM), parse_dates=["date"])16n_input_rows = {F_OLD: len(old), F_NEW: len(new), F_ASM: len(asm)}17issues = []1819for name, df in [(F_OLD, old), (F_NEW, new)]:20 assert not df.date.duplicated().any(), name21 full = pd.date_range(df.date.min(), df.date.max(), freq="MS")22 assert len(full) == len(df), f"gap in {name}"23 for d in df.loc[df.cpi.isna(), "date"]:24 issues.append({"issue": f"{name}: no index value reported for {d:%Y-%m} (empty cpi cell)",25 "handling": "Month treated as unavailable in that vintage; the older vintage does not cover it, so P is missing. "26 "Not interpolated; g_t for this month (and for t+12 if inside the window) reported as missing."})2728# Splice: most recent vintage in which the month is available (non-empty value).29o = old.dropna(subset=["cpi"]).assign(vintage_used=V_OLD).set_index("date")30n = new.dropna(subset=["cpi"]).assign(vintage_used=V_NEW).set_index("date")31lev = pd.concat([n, o[~o.index.isin(n.index)]]).sort_index()32ov = old.merge(new, on="date", suffixes=("_old", "_new")).dropna()33issues.append({"issue": f"Vintages overlap for {ov.date.min():%Y-%m}..{ov.date.max():%Y-%m} ({len(ov)} months) with differing levels "34 f"(max abs diff {np.abs(ov.cpi_new-ov.cpi_old).max():.3f} index points; seasonal-factor/revision differences)",35 "handling": "2026-10-28 vintage used for all overlap months per the most-recent-vintage rule; growth for 2014-01..2014-12 "36 "uses a 2026-vintage numerator and a 2016-vintage (2013) denominator, as the rule implies."})3738# Diagnose the hand-assembled file against the authoritative splice39a = asm.set_index("date")40full_rng = pd.date_range("1947-01-01", "2026-09-01", freq="MS")41missing_in_asm = [d for d in full_rng if d not in a.index]42extra_in_asm = [d for d in a.index if d not in full_rng]43if missing_in_asm:44 issues.append({"issue": f"{F_ASM}: months absent: {', '.join(f'{d:%Y-%m}' for d in missing_in_asm)}",45 "handling": "Assembled file not used for estimation; levels taken from the two vintage files."})46if extra_in_asm:47 issues.append({"issue": f"{F_ASM}: months beyond the sources' coverage: {', '.join(f'{d:%Y-%m}' for d in extra_in_asm)} "48 "(values appear shifted one month later from 2016-07 onward)",49 "handling": "Assembled file not used for estimation."})50cmp_ = a.join(lev[["cpi", "vintage_used"]], how="inner", rsuffix="_auth")51vint_wrong = int((cmp_.vintage != cmp_.vintage_used).sum())52val_wrong = int((~np.isclose(cmp_.cpi, cmp_.cpi_auth, equal_nan=True)).sum())53issues.append({"issue": f"{F_ASM}: of {len(cmp_)} months shared with the authoritative splice, {val_wrong} have a different level and "54 f"{vint_wrong} a different vintage label (2014-01..2016-06 taken from the older vintage; 2016-07 dropped and later values mis-dated)",55 "handling": "Assembled file read for diagnostics only; all estimates use the vintage files directly."})5657# Growth rates58lev = lev.reindex(pd.date_range(lev.index.min(), lev.index.max(), freq="MS"))59lag = lev.cpi.shift(12)60g = 100 * (lev.cpi / lag - 1)61win = pd.date_range(W0, W1, freq="MS")62tab = pd.DataFrame({"cpi": lev.cpi, "cpi_lag12": lag, "vintage_used": lev.vintage_used, "growth_12m": g}).reindex(win)63ok = tab.growth_12m.notna()64analytic = tab[ok].copy()65missing_months = [f"{d:%Y-%m}" for d in tab.index[~ok]]6667n2 = int(tab.cpi.notna().sum())68excl = [69 dict(step=1, rule="Window 2001-01..2026-09 (all calendar months)", n_before=len(win), n_after=len(win), n_removed=0,70 rationale="Start from every month of the target window"),71 dict(step=2, rule="Drop months with no index level P_t in the most recent covering vintage",72 n_before=len(win), n_after=n2, n_removed=len(win) - n2,73 rationale="No imputation allowed: " + (", ".join(f"{d:%Y-%m}" for d in tab.index[tab.cpi.isna()]) or "none")),74 dict(step=3, rule="Drop months with no index level P_{t-12}", n_before=n2, n_after=int(ok.sum()),75 n_removed=n2 - int(ok.sum()),76 rationale="No imputation allowed: " + (", ".join(f"{d:%Y-%m}" for d in tab.index[tab.cpi.notna() & tab.cpi_lag12.isna()]) or "none")),77]78pd.DataFrame(excl).to_csv(os.path.join(O, "exclusions.csv"), index=False)7980out_a = analytic.reset_index().rename(columns={"index": "date"})81out_a["date"] = out_a.date.dt.strftime("%Y-%m-%d")82out_a.to_csv(os.path.join(O, "analytic.csv"), index=False, float_format="%.10g")8384gs = analytic.growth_12m85def gm(m):86 v = tab.growth_12m.get(pd.Timestamp(m + "-01"))87 return None if v is None or pd.isna(v) else round(float(v), 10)88r = lambda x: round(float(x), 10)89last_month = lev.cpi.loc[:W1].dropna().index.max()90res = {91 "task_id": "F1-conf-01",92 "first_growth_month": f"{gs.index.min():%Y-%m}",93 "last_month": f"{last_month:%Y-%m}",94 "n_months_growth": int(ok.sum()),95 "n_months_missing_growth": int((~ok).sum()),96 "n_analysed": int(ok.sum()),97 "mean_growth_12m": r(gs.mean()),98 "mean_growth_12m_2001_2013": r(gs.loc[:"2013-12-01"].mean()),99 "mean_growth_12m_2014_2026": r(gs.loc["2014-01-01":].mean()),100 "growth_2014_06": gm("2014-06"), "growth_2016_07": gm("2016-07"), "growth_2020_05": gm("2020-05"),101 "growth_2022_06": gm("2022-06"), "growth_2026_09": gm("2026-09"),102 "max_growth_12m": r(gs.max()), "max_growth_month": f"{gs.idxmax():%Y-%m}",103 "min_growth_12m": r(gs.min()), "min_growth_month": f"{gs.idxmin():%Y-%m}",104 "feasibility": "complete" if ok.all() else "partial",105 "feasibility_reason": None if ok.all() else106 f"g_t not computable for {len(missing_months)} month(s) in the window ({', '.join(missing_months)}) because the source reports no index level; "107 "these months are excluded from all means and extremes. All summary estimands were computed.",108 "data_issues": issues,109 "n_input_rows": n_input_rows,110 "random_seed": None,111 "n_months_2001_2013": int(gs.loc[:"2013-12-01"].size),112 "n_months_2014_2026": int(gs.loc["2014-01-01":].size),113 "missing_growth_months": missing_months,114}115with open(os.path.join(O, "results.json"), "w") as f:116 json.dump(res, f, indent=2)117print(json.dumps({k: v for k, v in res.items() if k != "data_issues"}, indent=2))118