1"""F1-conf-01: 12-month CPIAUCSL growth from two ALFRED vintages.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Authoritative inputs are the two vintage files; the assembled file is read only for an audit."""4import json, os5import numpy as np6import pandas as pd78D, O = "data", "out"9OLD, NEW, ASM = "cpi_vintage_2016-07-15.csv", "cpi_vintage_2026-10-28.csv", "cpi_assembled.csv"10W0, W1 = pd.Timestamp("2001-01-01"), pd.Timestamp("2026-09-01")1112def load(f):13 d = pd.read_csv(os.path.join(D, f), dtype={"date": str})14 d["date"] = pd.to_datetime(d["date"])15 return d1617old, new, asm = load(OLD), load(NEW), load(ASM)18n_rows = {OLD: len(old), NEW: len(new), ASM: len(asm)}19issues = []2021# basic integrity checks on the vintage files22for name, d in ((OLD, old), (NEW, new)):23 assert not d["date"].duplicated().any(), name24 full = pd.date_range(d["date"].min(), d["date"].max(), freq="MS")25 assert len(full) == len(d), f"gap in date sequence of {name}"26 for dt in d.loc[d["cpi"].isna(), "date"]:27 issues.append({"issue": f"{name}: no CPI value published for {dt:%Y-%m} (empty cell)",28 "handling": "Left missing, not imputed; 12-month growth for that month (and for the month 12 later, if in the window) reported as missing."})2930# overlap / revisions between vintages31ov = old.merge(new, on="date", suffixes=("_old", "_new")).dropna()32nrev = int((ov["cpi_old"] != ov["cpi_new"]).sum())33issues.append({"issue": f"Vintages overlap for {ov['date'].min():%Y-%m}..{ov['date'].max():%Y-%m} ({len(ov)} months); {nrev} of them differ (revisions, e.g. seasonal-factor updates; max abs diff {np.abs(ov.cpi_old-ov.cpi_new).max():.3f} index points).",34 "handling": "Per the availability rule the 2026-10-28 vintage is used for every month it covers (2014-01 onward, including the overlap); 2016-07-15 vintage only before 2014-01."})35issues.append({"issue": "Growth rates for 2014-01..2014-12 divide a 2026-vintage level by a 2013 level from the 2016-07-15 vintage (mixed vintages), because the 2026 extract starts at 2014-01.",36 "handling": "Computed as specified by the rule (most recent available vintage per month); flagged as a limitation since 2013 levels may have been revised since 2016."})3738# combine: newest vintage where it covers the month (including months it covers with an empty cell -> missing, not back-filled)39o = old.assign(vintage_used="2016-07-15").set_index("date")40n = new.assign(vintage_used="2026-10-28").set_index("date")41comb = pd.concat([o[o.index < n.index.min()], n]).sort_index()42newer_missing = n.index[n["cpi"].isna()]43for dt in newer_missing:44 if dt in o.index:45 issues.append({"issue": f"{dt:%Y-%m} missing in 2026 vintage but present in 2016 vintage", "handling": "Not back-filled from the older vintage."})4647# audit of the hand-assembled file48a = asm.set_index("date")49exp = comb.reindex(a.index)50val_mis = a.index[~((a["cpi"] == exp["cpi"]) | (a["cpi"].isna() & exp["cpi"].isna()))]51vin_mis = a.index[a["vintage"].astype(str) != exp["vintage_used"].fillna("none")]52missing_dates = comb.index.difference(a.index)53extra_dates = a.index.difference(comb.index)54ov_old = a.loc[(a.index >= n.index.min()) & (a.index <= o.index.max())]55issues.append({"issue": f"cpi_assembled.csv contains assembly errors: overlap months {ov_old.index.min():%Y-%m}..{ov_old.index.max():%Y-%m} taken from the older 2016 vintage; months absent: {[f'{x:%Y-%m}' for x in missing_dates]}; spurious months not in any source: {[f'{x:%Y-%m}' for x in extra_dates]}; from 2016-08 onward values are shifted one month later than in the source (so its empty cell sits at 2025-11 instead of 2025-10); in total {len(val_mis)} rows have a value and {len(vin_mis)} rows a vintage label that disagree with the authoritative vintage files.",56 "handling": "The assembled file is not used for any computation; the series is rebuilt directly from the two vintage files (authoritative per TASK.md)."})5758# growth59s = comb["cpi"]60full = pd.date_range(s.index.min(), s.index.max(), freq="MS")61s = s.reindex(full)62lag = s.shift(12)63g = 100 * (s / lag - 1)64win = pd.date_range(W0, W1, freq="MS")65tab = pd.DataFrame({"cpi": s.reindex(win), "cpi_lag12": lag.reindex(win),66 "vintage_used": comb["vintage_used"].reindex(win), "growth_12m": g.reindex(win)})67ok = tab["growth_12m"].notna()68an = tab[ok]69miss = tab.index[~ok]70out = an.reset_index().rename(columns={"index": "date"})71out["date"] = out["date"].dt.strftime("%Y-%m-01")72out["growth_12m"] = out["growth_12m"].round(10)73out.to_csv(os.path.join(O, "analytic.csv"), index=False)7475no_level = tab.index[tab["cpi"].isna()]76no_lag = tab.index[tab["cpi"].notna() & tab["cpi_lag12"].isna()]77steps = [78 ("1", "Months 2001-01..2026-09 in the analysis window (one row per calendar month)", len(win), len(win), 0,79 "Starting frame: every month of the stated window, levels from most recent vintage covering the month."),80 ("2", "Drop months whose own index level P_t is missing in the source", len(win), len(win) - len(no_level), len(no_level),81 "No imputation allowed; missing: " + ", ".join(f"{x:%Y-%m}" for x in no_level)),82 ("3", "Drop months whose lagged level P_{t-12} is missing", len(win) - len(no_level), len(win) - len(no_level) - len(no_lag), len(no_lag),83 "No imputation allowed; missing: " + (", ".join(f"{x:%Y-%m}" for x in no_lag) or "none")),84]85pd.DataFrame(steps, columns=["step", "rule", "n_before", "n_after", "n_removed", "rationale"]).to_csv(os.path.join(O, "exclusions.csv"), index=False)86assert steps[-1][3] == len(an)8788gg = an["growth_12m"]89def gm(d):90 v = g.get(pd.Timestamp(d))91 return None if v is None or pd.isna(v) else float(v)92last_level = s.dropna().index.max()93res = {94 "task_id": "F1-conf-01",95 "first_growth_month": f"{an.index.min():%Y-%m}",96 "last_month": f"{min(last_level, W1):%Y-%m}",97 "n_months_growth": int(len(an)),98 "n_months_missing_growth": int(len(miss)),99 "n_analysed": int(len(an)),100 "mean_growth_12m": float(gg.mean()),101 "mean_growth_12m_2001_2013": float(gg[:"2013-12-01"].mean()),102 "mean_growth_12m_2014_2026": float(gg["2014-01-01":].mean()),103 "growth_2014_06": gm("2014-06-01"), "growth_2016_07": gm("2016-07-01"),104 "growth_2020_05": gm("2020-05-01"), "growth_2022_06": gm("2022-06-01"),105 "growth_2026_09": gm("2026-09-01"),106 "max_growth_12m": float(gg.max()), "max_growth_month": f"{gg.idxmax():%Y-%m}",107 "min_growth_12m": float(gg.min()), "min_growth_month": f"{gg.idxmin():%Y-%m}",108 "feasibility": "complete",109 "feasibility_reason": None,110 "data_issues": issues,111 "n_input_rows": n_rows,112 "random_seed": None,113 "missing_growth_months": [f"{x:%Y-%m}" for x in miss],114 "n_2001_2013": int(len(gg[:"2013-12-01"])), "n_2014_2026": int(len(gg["2014-01-01":])),115}116# sensitivity (not the estimand): old-vintage-only growth in overlap, and naive use of the assembled file117so = old.set_index("date")["cpi"]; go = 100 * (so / so.shift(12) - 1)118sa = a["cpi"].reindex(pd.date_range(a.index.min(), a.index.max(), freq="MS")); ga = (100 * (sa / sa.shift(12) - 1)).reindex(win).dropna()119res["sensitivity"] = {120 "growth_2014_06_vintage2016": float(go["2014-06-01"]),121 "mean_growth_2014_01_2016_06_vintage2016": float(go["2014-01-01":"2016-06-01"].mean()),122 "mean_growth_2014_01_2016_06_used": float(gg["2014-01-01":"2016-06-01"].mean()),123 "assembled_file_naive_mean_growth_12m": float(ga.mean()),124 "assembled_file_naive_n_growth": int(len(ga)),125 "assembled_file_naive_growth_2026_09": float(ga.get(pd.Timestamp("2026-09-01"), np.nan)),126}127with open(os.path.join(O, "results.json"), "w") as f:128 json.dump(res, f, indent=2)129print(json.dumps({k: v for k, v in res.items() if k != "data_issues"}, indent=2))130for i in issues: print("-", i["issue"])131