1"""F1-conf-01: 12-month CPI inflation from two ALFRED vintages.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Reads only data/; writes out/results.json, out/analytic.csv, out/exclusions.csv. No randomness."""4import json5import numpy as np6import pandas as pd78D = "data/"9F_OLD, F_NEW, F_ASM = "cpi_vintage_2016-07-15.csv", "cpi_vintage_2026-10-28.csv", "cpi_assembled.csv"10W0, W1 = pd.Timestamp("2001-01-01"), pd.Timestamp("2026-09-01")1112old = pd.read_csv(D + F_OLD, parse_dates=["date"])13new = pd.read_csv(D + F_NEW, parse_dates=["date"])14asm = pd.read_csv(D + F_ASM, parse_dates=["date"], dtype={"vintage": str})15n_input_rows = {F_OLD: len(old), F_NEW: len(new), F_ASM: len(asm)}1617issues = []18for name, df in [(F_OLD, old), (F_NEW, new)]:19 assert not df.date.duplicated().any(), name20 full = pd.date_range(df.date.min(), df.date.max(), freq="MS")21 assert len(full) == len(df), f"gap in {name}"22miss_new = new.loc[new.cpi.isna(), "date"].dt.strftime("%Y-%m").tolist()23if miss_new:24 issues.append({"issue": f"{F_NEW} has no index value (empty cpi) for {', '.join(miss_new)}",25 "handling": "Not imputed or back-filled from another source; growth for that month (and for the month 12 later) is reported as missing and counted."})2627# Overlap / revisions28ov = old.merge(new, on="date", suffixes=("_old", "_new"))29ndiff = int((ov.cpi_old != ov.cpi_new).sum())30if ndiff:31 issues.append({"issue": f"The two vintages overlap for {len(ov)} months ({ov.date.min():%Y-%m} to {ov.date.max():%Y-%m}) and differ in {ndiff} of them (revisions; max abs diff {np.abs(ov.cpi_old-ov.cpi_new).max():.3f} index points)",32 "handling": "Per the availability rule, the 2026-10-28 vintage value is used for every month it covers; 2016-07-15 values only before 2014-01. Growth rates for 2014-01..2014-12 therefore combine a 2026-vintage numerator with a 2016-vintage base."})3334# Most-recent-vintage series35lv = pd.concat([old.assign(vintage_used="2016-07-15"), new.assign(vintage_used="2026-10-28")])36lv = lv.sort_values(["date", "vintage_used"]).drop_duplicates("date", keep="last").set_index("date")37months = pd.date_range(lv.index.min(), lv.index.max(), freq="MS")38lv = lv.reindex(months)39assert lv.vintage_used.notna().all()4041# Audit assembled file against authoritative series42a = asm.set_index("date")43asm_missing_months = [m.strftime("%Y-%m") for m in months if m not in a.index]44asm_extra = [m.strftime("%Y-%m") for m in a.index if m not in lv.index]45cmp = a.join(lv[["cpi", "vintage_used"]], rsuffix="_auth", how="inner")46mism = cmp[~((cmp.cpi == cmp.cpi_auth) | (cmp.cpi.isna() & cmp.cpi_auth.isna()))]47oldlab = cmp[(cmp.index >= new.date.min()) & (cmp.vintage != "2026-10-28")]48if len(mism) or asm_missing_months or asm_extra:49 issues.append({"issue": (f"{F_ASM} contains assembly errors: months absent {asm_missing_months}; months beyond the sources {asm_extra}; "50 f"{len(mism)} months whose value differs from the most-recent-vintage value ({len(oldlab)} months 2014-01.. taken from the 2016 vintage although the 2026 vintage covers them; "51 f"values after the missing month shifted one month later)"),52 "handling": "File read only for auditing; all computations use the two authoritative vintage files directly."})53issues.append({"issue": "README SHA-256 values refer to the raw downloads before reformatting, so they cannot be matched against the provided files",54 "handling": "Recorded SHA-256 of provided files in out/state/data_audit.md; values not otherwise verifiable offline."})5556# Growth57lv["cpi_lag12"] = lv.cpi.shift(12)58lv["growth_12m"] = 100 * (lv.cpi / lv.cpi_lag12 - 1)59win = lv.loc[W0:W1].copy()60n_window = len(win)61ok = win[win.growth_12m.notna()].copy()62missing_g = win.index[win.growth_12m.isna()].strftime("%Y-%m").tolist()6364# Exclusion ledger65n_all = len(lv)66rows = [67 ("1", "Combined most-recent-vintage monthly series 1947-01..2026-09 restricted to window 2001-01..2026-09", n_all, n_window,68 "Window defined by the task"),69 ("2", "Drop window months without computable g_t (P_t or P_{t-12} missing in sources)", n_window, len(ok),70 f"No interpolation/imputation allowed; missing: {', '.join(missing_g) if missing_g else 'none'}"),71]72excl = pd.DataFrame([{"step": s, "rule": r, "n_before": b, "n_after": a_, "n_removed": b - a_, "rationale": ra} for s, r, b, a_, ra in rows])73excl.to_csv("out/exclusions.csv", index=False)7475out = ok.reset_index().rename(columns={"index": "date"})76out["date"] = out.date.dt.strftime("%Y-%m-%d")77out[["date", "cpi", "cpi_lag12", "vintage_used", "growth_12m"]].to_csv("out/analytic.csv", index=False, float_format="%.10g")7879g = ok.growth_12m80def gm(s): return None if pd.isna(lv.growth_12m.get(pd.Timestamp(s))) else round(float(lv.growth_12m[pd.Timestamp(s)]), 6)81last_level = lv.loc[:W1].cpi.dropna().index.max()82res = {83 "task_id": "F1-conf-01",84 "first_growth_month": ok.index.min().strftime("%Y-%m"),85 "last_month": last_level.strftime("%Y-%m"),86 "n_months_window": n_window,87 "n_months_growth": int(len(ok)),88 "n_months_missing_growth": int(n_window - len(ok)),89 "missing_growth_months": missing_g,90 "n_analysed": int(len(ok)),91 "mean_growth_12m": round(float(g.mean()), 6),92 "mean_growth_12m_2001_2013": round(float(g.loc["2001-01-01":"2013-12-01"].mean()), 6),93 "mean_growth_12m_2014_2026": round(float(g.loc["2014-01-01":"2026-09-01"].mean()), 6),94 "n_growth_2001_2013": int(g.loc["2001-01-01":"2013-12-01"].size),95 "n_growth_2014_2026": int(g.loc["2014-01-01":"2026-09-01"].size),96 "growth_2014_06": gm("2014-06-01"),97 "growth_2016_07": gm("2016-07-01"),98 "growth_2020_05": gm("2020-05-01"),99 "growth_2022_06": gm("2022-06-01"),100 "growth_2026_09": gm("2026-09-01"),101 "max_growth_12m": round(float(g.max()), 6),102 "max_growth_month": g.idxmax().strftime("%Y-%m"),103 "min_growth_12m": round(float(g.min()), 6),104 "min_growth_month": g.idxmin().strftime("%Y-%m"),105 "sd_growth_12m": round(float(g.std(ddof=1)), 6),106 "feasibility": "complete" if not missing_g else "partial",107 "feasibility_reason": None if not missing_g else108 f"g_t not computable for {', '.join(missing_g)}: the 2026-10-28 vintage reports no CPI level for 2025-10 and imputation is not allowed; all summaries use the {len(ok)} computable months.",109 "data_issues": issues,110 "n_input_rows": n_input_rows,111 "random_seed": None,112}113with open("out/results.json", "w") as f:114 json.dump(res, f, indent=2)115print(f"n_analysed={res['n_analysed']} missing={missing_g} mean={res['mean_growth_12m']:.4f}")116