1"""F1-conf-01: 12-month CPI inflation from two ALFRED vintages.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Reads only data/; writes out/results.json, out/analytic.csv, out/exclusions.csv. No randomness."""4import json, os5import numpy as np6import pandas as pd78D, O = "data", "out"9F_OLD, F_NEW, F_ASM = "cpi_vintage_2016-07-15.csv", "cpi_vintage_2026-10-28.csv", "cpi_assembled.csv"10W0, W1 = pd.Timestamp("2001-01-01"), pd.Timestamp("2026-09-01")1112old = pd.read_csv(os.path.join(D, F_OLD), parse_dates=["date"])13new = pd.read_csv(os.path.join(D, F_NEW), parse_dates=["date"])14asm = pd.read_csv(os.path.join(D, F_ASM), parse_dates=["date"]) # read for audit only15n_rows = {F_OLD: len(old), F_NEW: len(new), F_ASM: len(asm)}16issues = []1718# --- sanity checks on authoritative vintages19for name, df in [(F_OLD, old), (F_NEW, new)]:20 assert not df.date.duplicated().any(), name21 assert (df.date.dt.day == 1).all(), name22 full = pd.date_range(df.date.min(), df.date.max(), freq="MS")23 assert len(full) == len(df), f"gap in date grid of {name}"2425new_missing = new.loc[new.cpi.isna(), "date"].dt.strftime("%Y-%m").tolist()26if new_missing:27 issues.append({"issue": f"{F_NEW} has an empty cpi cell (no published value) for {', '.join(new_missing)}",28 "handling": "Not imputed and not filled from the older vintage (the 2016 vintage does not cover it); "29 "g_t for 2025-10 is missing and counted (the dependent month 2026-10 lies outside the window and outside source coverage)."})3031# --- audit of the hand-assembled file against the vintages32a = asm.set_index("date")33exp_old = old.set_index("date").cpi34exp_new = new.set_index("date").cpi35overlap = exp_old.index.intersection(exp_new.index)36asm_ov = a.loc[a.index.intersection(overlap)]37n_old_in_overlap = int((asm_ov.vintage == "2016-07-15").sum())38if n_old_in_overlap:39 issues.append({"issue": f"{F_ASM}: {n_old_in_overlap} overlap months ({overlap.min():%Y-%m}..{overlap.max():%Y-%m}) take the 2016-07-15 value instead of the most recent (2026-10-28) vintage",40 "handling": "Assembled file not used; series rebuilt from the two vintage files with the most-recent-vintage rule."})41missing_in_asm = [d for d in exp_new.index if d not in a.index]42if missing_in_asm:43 issues.append({"issue": f"{F_ASM}: months present in {F_NEW} are absent: {', '.join(d.strftime('%Y-%m') for d in missing_in_asm)}",44 "handling": "Assembled file not used."})45asm_new = a[a.vintage == "2026-10-28"]46n_mis = int(sum(1 for d, r in asm_new.iterrows()47 if d not in exp_new.index or not ((pd.isna(r.cpi) and pd.isna(exp_new[d])) or np.isclose(r.cpi, exp_new[d]))))48if n_mis:49 beyond = [d.strftime('%Y-%m') for d in asm_new.index if d > new.date.max()]50 issues.append({"issue": f"{F_ASM}: {n_mis} rows labelled 2026-10-28 do not match {F_NEW} at the same date (values appear shifted by one month from 2016-08 onward; dates beyond source coverage: {beyond}; missing value placed at 2025-11 instead of 2025-10)",51 "handling": "Assembled file not used."})5253# --- build series: newest vintage where it covers the month, else old vintage54s_new = new.set_index("date")55s_old = old.set_index("date")56covered_new = s_new.index # months covered by the 2026 vintage extract (incl. empty cell)57rows = []58for d in pd.date_range(old.date.min(), new.date.max(), freq="MS"):59 if d in covered_new:60 rows.append((d, s_new.cpi[d], "2026-10-28"))61 elif d in s_old.index:62 rows.append((d, s_old.cpi[d], "2016-07-15"))63 else:64 rows.append((d, np.nan, None))65ser = pd.DataFrame(rows, columns=["date", "cpi", "vintage_used"]).set_index("date")66assert ser.vintage_used.notna().all()67ser["cpi_lag12"] = ser.cpi.shift(12) # contiguous monthly grid, so shift(12) = t-1268ser["growth_12m"] = 100.0 * (ser.cpi / ser.cpi_lag12 - 1.0)6970win = ser.loc[W0:W1].copy()71n_win = len(win)72step1 = win[win.cpi.notna()]73step2 = step1[step1.cpi_lag12.notna()]74an = step2.copy()75excl = pd.DataFrame([76 dict(step=1, rule="window 2001-01..2026-09 drop months with no index level P_t in the sources",77 n_before=n_win, n_after=len(step1), n_removed=n_win - len(step1),78 rationale="no imputation of missing levels (empty cell in 2026-10-28 vintage)"),79 dict(step=2, rule="drop months whose lagged level P_{t-12} is missing",80 n_before=len(step1), n_after=len(step2), n_removed=len(step1) - len(step2),81 rationale="g_t not computable without P_{t-12}; no imputation"),82])83excl.to_csv(os.path.join(O, "exclusions.csv"), index=False)8485out = an.reset_index()[["date", "cpi", "cpi_lag12", "vintage_used", "growth_12m"]]86out["date"] = out.date.dt.strftime("%Y-%m-01")87out.to_csv(os.path.join(O, "analytic.csv"), index=False, float_format="%.10g")8889g = an.growth_12m90def gm(m):91 d = pd.Timestamp(m + "-01")92 return float(round(g[d], 10)) if d in g.index else None9394missing_months = win.index[win.growth_12m.isna()].strftime("%Y-%m").tolist()95last_level = ser.loc[:W1].cpi.dropna().index.max()96res = {97 "task_id": "F1-conf-01",98 "first_growth_month": g.index.min().strftime("%Y-%m"),99 "last_month": last_level.strftime("%Y-%m"),100 "n_months_growth": int(len(g)),101 "n_months_missing_growth": int(n_win - len(g)),102 "n_analysed": int(len(g)),103 "mean_growth_12m": float(round(g.mean(), 10)),104 "mean_growth_12m_2001_2013": float(round(g.loc["2001-01-01":"2013-12-01"].mean(), 10)),105 "mean_growth_12m_2014_2026": float(round(g.loc["2014-01-01":"2026-09-01"].mean(), 10)),106 "growth_2014_06": gm("2014-06"), "growth_2016_07": gm("2016-07"), "growth_2020_05": gm("2020-05"),107 "growth_2022_06": gm("2022-06"), "growth_2026_09": gm("2026-09"),108 "max_growth_12m": float(round(g.max(), 10)), "max_growth_month": g.idxmax().strftime("%Y-%m"),109 "min_growth_12m": float(round(g.min(), 10)), "min_growth_month": g.idxmin().strftime("%Y-%m"),110 "feasibility": "complete" if not missing_months else "partial",111 "feasibility_reason": None if not missing_months else112 f"g_t not computable for {len(missing_months)} month(s) of the {n_win}-month window ({', '.join(missing_months)}) because the most recent vintage publishes no index level for 2025-10; all other estimands computed.",113 "data_issues": issues,114 "n_input_rows": n_rows,115 "random_seed": None,116 "n_window_months": int(n_win),117 "missing_growth_months": missing_months,118 "n_growth_2001_2013": int(len(g.loc["2001-01-01":"2013-12-01"])),119 "n_growth_2014_2026": int(len(g.loc["2014-01-01":"2026-09-01"])),120 "sd_growth_12m": float(round(g.std(ddof=1), 10)),121}122with open(os.path.join(O, "results.json"), "w") as f:123 json.dump(res, f, indent=2)124print(json.dumps({k: res[k] for k in ["n_analysed", "mean_growth_12m", "max_growth_month", "min_growth_month", "feasibility"]}))125