1"""F1-conf-01: 12-month CPI inflation from two ALFRED vintages.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py"""3import json, os4import numpy as np, pandas as pd56D, O = "data", "out"7os.makedirs(O, exist_ok=True)8f_old, f_new, f_asm = "cpi_vintage_2016-07-15.csv", "cpi_vintage_2026-10-28.csv", "cpi_assembled.csv"9old = pd.read_csv(os.path.join(D, f_old), dtype={"cpi": float})10new = pd.read_csv(os.path.join(D, f_new), dtype={"cpi": float})11asm = pd.read_csv(os.path.join(D, f_asm), dtype={"cpi": float, "vintage": str})12n_input = {f_old: len(old), f_new: len(new), f_asm: len(asm)}13for d in (old, new, asm):14 d["date"] = pd.to_datetime(d["date"])15issues = []1617# basic integrity checks on the authoritative vintage files18for name, d in ((f_old, old), (f_new, new)):19 assert not d["date"].duplicated().any(), name20 assert d["date"].is_monotonic_increasing, name21 full = pd.date_range(d["date"].min(), d["date"].max(), freq="MS")22 assert len(full) == len(d), f"{name}: gap in date rows"23 for dt in d.loc[d["cpi"].isna(), "date"]:24 issues.append({"issue": f"{name}: no CPI value reported for {dt:%Y-%m} (empty cell)",25 "handling": "Treated as missing; not interpolated or imputed. No value exists in the older vintage either "26 "(it ends 2016-06), so the index level for this month is missing and g_t for this month "27 "(and for t+12, which lies outside the window) cannot be computed."})2829# splice: newest vintage where it covers the month (row present with a value), else 2016 vintage30new_ok = new.dropna(subset=["cpi"]).assign(vintage="2026-10-28")31new_missing = set(new.loc[new["cpi"].isna(), "date"])32old_use = old[~old["date"].isin(new["date"])].assign(vintage="2016-07-15")33# months in overlap with missing new value: the most recent vintage in which the month is AVAILABLE could be the old one34fallback = old[old["date"].isin(new_missing)].dropna(subset=["cpi"]).assign(vintage="2016-07-15")35lev = pd.concat([old_use, fallback, new_ok]).sort_values("date").set_index("date")36assert not lev.index.duplicated().any()3738ov = old.merge(new, on="date", suffixes=("_old", "_new")).dropna()39dif = ov["cpi_new"] - ov["cpi_old"]40issues.append({"issue": f"Vintages overlap for {len(ov)} months (2014-01..2016-06) and disagree (revisions incl. seasonal-factor "41 f"updates): new-old ranges {dif.min():.3f} to {dif.max():.3f} index points.",42 "handling": "Per the availability rule the 2026-10-28 value is used for every overlapping month. Consequently "43 "g_t for 2014-01..2014-12 divides a 2026-vintage level by a 2016-vintage level (2013 months exist "44 "only in the old extract); this cross-vintage mixing is flagged in the report."})4546# audit the hand-assembled file against the authoritative splice (documentation only; not used for estimates)47ref = lev["cpi"].reindex(pd.date_range("1947-01-01", "2026-09-01", freq="MS"))48a = asm.set_index("date")49wrong_vint = a.loc["2014-01-01":"2016-06-01"]50n_wv = int((wrong_vint["vintage"] == "2016-07-15").sum())51if n_wv:52 issues.append({"issue": f"cpi_assembled.csv uses the 2016-07-15 vintage for {n_wv} overlapping months 2014-01..2016-06 "53 "instead of the most recent (2026-10-28) vintage.",54 "handling": "Assembled file not used; levels rebuilt from the two vintage files."})55missing_rows = ref.index.difference(a.index)56extra_rows = a.index.difference(ref.index)57if len(missing_rows) or len(extra_rows):58 issues.append({"issue": f"cpi_assembled.csv lacks month(s) {[f'{x:%Y-%m}' for x in missing_rows]} and contains month(s) "59 f"beyond the sources {[f'{x:%Y-%m}' for x in extra_rows]}: after the dropped month the 2026-vintage "60 "values are shifted one month later (off-by-one dating).",61 "handling": "Assembled file not used; levels rebuilt from the two vintage files."})62common = ref.index.intersection(a.index)63mism = (~np.isclose(a.loc[common, "cpi"], ref.loc[common], atol=1e-9) &64 ~(a.loc[common, "cpi"].isna() & ref.loc[common].isna()))65mism |= a.loc[common, "cpi"].isna() != ref.loc[common].isna()66issues.append({"issue": f"cpi_assembled.csv values disagree with the authoritative splice in {int(mism.sum())} of {len(common)} common months.",67 "handling": "Assembled file treated as unreliable and used only for this audit."})6869# growth rates70win = pd.date_range("2001-01-01", "2026-09-01", freq="MS")71P = lev["cpi"].reindex(pd.date_range("1947-01-01", "2026-09-01", freq="MS"))72V = lev["vintage"].reindex(P.index)73lag = P.shift(12)74g = 100 * (P / lag - 1)75df = pd.DataFrame({"cpi": P, "cpi_lag12": lag, "vintage_used": V, "growth_12m": g}).loc[win]76n_win = len(df)77step1 = df.dropna(subset=["cpi"])78step2 = step1.dropna(subset=["cpi_lag12"])79an = step280missing_months = [f"{x:%Y-%m}" for x in df.index[df["growth_12m"].isna()]]8182ex = pd.DataFrame([83 {"step": 1, "rule": "Months 2001-01..2026-09 of the spliced series", "n_before": n_win, "n_after": n_win, "n_removed": 0,84 "rationale": "Analysis window defined in TASK.md (one row per calendar month)."},85 {"step": 2, "rule": "Drop months whose index level P_t is missing in all vintages", "n_before": n_win, "n_after": len(step1),86 "n_removed": n_win - len(step1), "rationale": "No interpolation/imputation allowed; 2025-10 has no value in the 2026-10-28 vintage."},87 {"step": 3, "rule": "Drop months whose lagged level P_{t-12} is missing", "n_before": len(step1), "n_after": len(step2),88 "n_removed": len(step1) - len(step2), "rationale": "g_t requires both levels; lag of 2026-10 would be 2025-10 but 2026-10 is outside the window."},89])90ex.to_csv(os.path.join(O, "exclusions.csv"), index=False)9192out = an.reset_index().rename(columns={"index": "date"})93out["date"] = out["date"].dt.strftime("%Y-%m-01")94out[["date", "cpi", "cpi_lag12", "vintage_used", "growth_12m"]].to_csv(os.path.join(O, "analytic.csv"), index=False, float_format="%.10g")9596gg = an["growth_12m"]97def gm(m):98 v = df.loc[pd.Timestamp(m + "-01"), "growth_12m"]99 return None if pd.isna(v) else round(float(v), 6)100# sensitivity (not a required estimand): 2014 growth using only the 2016-07-15 vintage (single-vintage ratio)101Po = old.set_index("date")["cpi"]102go = 100 * (Po / Po.shift(12) - 1)103sens_2014 = go.loc["2014-01-01":"2014-12-01"].mean()104sens_2014_06 = go.loc["2014-06-01"]105last_level = P.dropna().index.max()106res = {107 "task_id": "F1-conf-01",108 "first_growth_month": f"{an.index.min():%Y-%m}",109 "last_month": f"{last_level:%Y-%m}",110 "n_months_growth": int(len(an)),111 "n_months_missing_growth": int(n_win - len(an)),112 "n_analysed": int(len(an)),113 "mean_growth_12m": round(float(gg.mean()), 6),114 "mean_growth_12m_2001_2013": round(float(gg.loc["2001-01-01":"2013-12-01"].mean()), 6),115 "mean_growth_12m_2014_2026": round(float(gg.loc["2014-01-01":"2026-09-01"].mean()), 6),116 "growth_2014_06": gm("2014-06"), "growth_2016_07": gm("2016-07"), "growth_2020_05": gm("2020-05"),117 "growth_2022_06": gm("2022-06"), "growth_2026_09": gm("2026-09"),118 "max_growth_12m": round(float(gg.max()), 6), "max_growth_month": f"{gg.idxmax():%Y-%m}",119 "min_growth_12m": round(float(gg.min()), 6), "min_growth_month": f"{gg.idxmin():%Y-%m}",120 "feasibility": "complete" if not missing_months else "partial",121 "feasibility_reason": None if not missing_months else122 f"g_t not computable for {missing_months}: the 2026-10-28 vintage reports no CPI value for 2025-10 and imputation is "123 "not allowed. All other window months and all summary statistics are computed (means over available months).",124 "data_issues": issues,125 "n_input_rows": n_input,126 "random_seed": None,127 "sensitivity_mean_growth_2014_mixed_vintage": round(float(gg.loc["2014-01-01":"2014-12-01"].mean()), 6),128 "sensitivity_mean_growth_2014_old_vintage_only": round(float(sens_2014), 6),129 "sensitivity_growth_2014_06_old_vintage_only": round(float(sens_2014_06), 6),130}131with open(os.path.join(O, "results.json"), "w") as f:132 json.dump(res, f, indent=2)133print(json.dumps({k: v for k, v in res.items() if k != "data_issues"}, indent=2))134