1"""F1-conf-01: 12-month CPI inflation from two ALFRED vintages.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Reads only data/; writes out/results.json, out/analytic.csv, out/exclusions.csv. No randomness."""4import json5import numpy as np6import pandas as pd78OLD, NEW, ASM = "cpi_vintage_2016-07-15.csv", "cpi_vintage_2026-10-28.csv", "cpi_assembled.csv"9W0, W1 = pd.Timestamp("2001-01-01"), pd.Timestamp("2026-09-01")101112def load(f):13 d = pd.read_csv("data/" + f, dtype={"date": str})14 d["date"] = pd.to_datetime(d["date"])15 return d161718old, new, asm = load(OLD), load(NEW), load(ASM)19for d in (old, new):20 assert not d["date"].duplicated().any()21n_input_rows = {OLD: len(old), NEW: len(new), ASM: len(asm)}2223# Latest-vintage rule: a month is "covered" by the 2026 vintage if it appears in that extract24# (rows 2014-01..2026-09). An empty cell there means the source reports no value; we do NOT25# fall back to the older vintage (it does not cover 2025-10 anyway) and do not impute.26o = old.set_index("date")["cpi"]27nw = new.set_index("date")["cpi"]28months = pd.date_range(min(o.index.min(), nw.index.min()), max(o.index.max(), nw.index.max()), freq="MS")29level = pd.Series(np.nan, index=months)30vint = pd.Series(None, index=months, dtype=object)31level.loc[o.index] = o.values32vint.loc[o.index] = "2016-07-15"33level.loc[nw.index] = nw.values34vint.loc[nw.index] = "2026-10-28"3536window = pd.date_range(W0, W1, freq="MS")37lag = level.shift(12)38g = 100.0 * (level / lag - 1.0)39df = pd.DataFrame({"cpi": level, "cpi_lag12": lag, "vintage_used": vint, "growth_12m": g}).loc[window]40ok = df["growth_12m"].notna()41analytic = df[ok].copy()42missing_months = [d.strftime("%Y-%m") for d in df.index[~ok]]4344# Exclusion ledger45excl = [46 dict(step=1, rule="calendar months 2001-01..2026-09 (window)", n_before=len(window), n_after=len(window),47 n_removed=0, rationale="starting set: every month of the estimation window"),48]49n_lvl = int(df["cpi"].notna().sum())50excl.append(dict(step=2, rule="drop months with no index level P_t in latest covering vintage",51 n_before=len(window), n_after=n_lvl, n_removed=len(window) - n_lvl,52 rationale="no interpolation/imputation allowed (e.g. 2025-10 empty in 2026-10-28 vintage)"))53n_ok = int(ok.sum())54excl.append(dict(step=3, rule="drop months with no lagged level P_{t-12}", n_before=n_lvl, n_after=n_ok,55 n_removed=n_lvl - n_ok, rationale="g_t requires both P_t and P_{t-12}"))56pd.DataFrame(excl).to_csv("out/exclusions.csv", index=False)5758out_csv = analytic.reset_index().rename(columns={"index": "date"})59out_csv["date"] = out_csv["date"].dt.strftime("%Y-%m-01")60out_csv.to_csv("out/analytic.csv", index=False, float_format="%.10g")6162G = analytic["growth_12m"]636465def gm(m):66 t = pd.Timestamp(m + "-01")67 v = G.get(t, np.nan)68 return None if pd.isna(v) else round(float(v), 10)697071def r(x):72 return round(float(x), 10)737475# Data issues detected (computed, not hard-coded where possible)76issues = []77empty_new = [d.strftime("%Y-%m") for d in nw.index[nw.isna()]]78if empty_new:79 issues.append({"issue": f"{NEW}: empty cpi for {', '.join(empty_new)} (source reports no value)",80 "handling": "level treated as missing, not imputed; g_t for that month reported missing "81 "(and any month t whose t-12 is that month; inside the window: none)"})82ov = o.index.intersection(nw.index)83dif = (nw[ov] - o[ov]).dropna()84if len(dif) and (dif.abs() > 1e-9).any():85 issues.append({"issue": f"vintages overlap for {len(ov)} months ({ov.min():%Y-%m}..{ov.max():%Y-%m}) with revised values "86 f"(max abs diff {dif.abs().max():.3f} index points)",87 "handling": "2026-10-28 vintage used for overlap months per latest-vintage rule"})88a = asm.set_index("date")89a_old = a[a["vintage"].astype(str) == "2016-07-15"]90a_new = a[a["vintage"].astype(str) == "2026-10-28"]91asm_missing = [d.strftime("%Y-%m") for d in months if d not in a.index]92if (a_old.index >= nw.index.min()).any():93 issues.append({"issue": f"{ASM}: months {nw.index.min():%Y-%m}..{a_old.index.max():%Y-%m} taken from the older 2016-07-15 vintage although the 2026-10-28 vintage covers them",94 "handling": "assembled file not used; levels taken directly from the vintage files"})95if asm_missing:96 issues.append({"issue": f"{ASM}: month(s) {', '.join(asm_missing)} absent",97 "handling": "assembled file not used; levels taken directly from the vintage files"})98nw_shift = nw.shift(1, freq="MS")99common = a_new.index.intersection(nw_shift.index)100same_shift = np.allclose(a_new.loc[common, "cpi"].fillna(-1), nw_shift.loc[common].fillna(-1))101if same_shift:102 issues.append({"issue": f"{ASM}: rows labelled 2026-10-28 ({a_new.index.min():%Y-%m}..{a_new.index.max():%Y-%m}) are shifted one month late (value at month m equals the 2026-10-28 vintage value for m-1); file extends to {a.index.max():%Y-%m}, beyond the source",103 "handling": "assembled file not used; levels taken directly from the vintage files"})104105last_month = level.loc[window].dropna().index.max().strftime("%Y-%m")106imax, imin = G.idxmax(), G.idxmin()107n_miss = len(window) - n_ok108res = {109 "task_id": "F1-conf-01",110 "first_growth_month": G.index.min().strftime("%Y-%m"),111 "last_month": last_month,112 "n_months_growth": n_ok,113 "n_months_missing_growth": n_miss,114 "n_analysed": n_ok,115 "mean_growth_12m": r(G.mean()),116 "mean_growth_12m_2001_2013": r(G[:"2013-12-01"].mean()),117 "mean_growth_12m_2014_2026": r(G["2014-01-01":].mean()),118 "growth_2014_06": gm("2014-06"),119 "growth_2016_07": gm("2016-07"),120 "growth_2020_05": gm("2020-05"),121 "growth_2022_06": gm("2022-06"),122 "growth_2026_09": gm("2026-09"),123 "max_growth_12m": r(G.max()),124 "max_growth_month": imax.strftime("%Y-%m"),125 "min_growth_12m": r(G.min()),126 "min_growth_month": imin.strftime("%Y-%m"),127 "feasibility": "complete" if n_miss == 0 else "partial",128 "feasibility_reason": None if n_miss == 0 else129 f"g_t not computable for {n_miss} month(s) of the window ({', '.join(missing_months)}): index level absent in the source; not imputed per the task rules",130 "missing_growth_months": missing_months,131 "n_months_window": len(window),132 "n_months_growth_2001_2013": int(G[:"2013-12-01"].notna().sum()),133 "n_months_growth_2014_2026": int(G["2014-01-01":].notna().sum()),134 "n_months_mixed_vintage": int((vint.shift(12).loc[analytic.index] != analytic["vintage_used"]).sum()),135 "sd_growth_12m": r(G.std(ddof=1)),136 "data_issues": issues,137 "n_input_rows": n_input_rows,138 "random_seed": None,139}140with open("out/results.json", "w") as f:141 json.dump(res, f, indent=2)142print(json.dumps({k: res[k] for k in ["n_analysed", "n_months_missing_growth", "mean_growth_12m", "feasibility"]}))143