1"""F1-conf-01: 12-month CPI inflation from two vintages. Run from workspace root."""2import json, os3import numpy as np, pandas as pd45D = "data"; O = "out"6F_OLD, F_NEW, F_AS = "cpi_vintage_2016-07-15.csv", "cpi_vintage_2026-10-28.csv", "cpi_assembled.csv"7W0, W1 = pd.Timestamp("2001-01-01"), pd.Timestamp("2026-09-01")89def load(f):10 d = pd.read_csv(os.path.join(D, f), dtype={"date": str})11 d["date"] = pd.to_datetime(d["date"]); return d1213old, new = load(F_OLD), load(F_NEW)14asm = load(F_AS); asm["vintage"] = asm["vintage"].astype(str)15n_rows = {F_OLD: len(old), F_NEW: len(new), F_AS: len(asm)}16issues = []1718# --- detected issues ---19miss_new = new[new.cpi.isna()].date.dt.strftime("%Y-%m").tolist()20if miss_new:21 issues.append({"issue": f"Vintage 2026-10-28 has no CPI value (empty cell) for {', '.join(miss_new)}.",22 "handling": "Not imputed or back-filled from the older vintage (which does not cover it anyway); g_t for that month reported missing and counted; g_t twelve months later lies outside the window."})23ov = old.merge(new, on="date", suffixes=("_old", "_new"))24if len(ov):25 issues.append({"issue": f"The two vintages overlap for {len(ov)} months ({ov.date.min():%Y-%m}..{ov.date.max():%Y-%m}) with revised values (max abs difference {np.nanmax(np.abs(ov.cpi_old-ov.cpi_new)):.3f} index points).",26 "handling": "Values from the most recent vintage (2026-10-28) used for overlapping months, per the information rule."})27 issues.append({"issue": "Growth rates for 2014-01..2014-12 divide a 2026-10-28-vintage level by a 2016-07-15-vintage level (different revision/seasonal-adjustment states).",28 "handling": "Kept as required by the 'most recent vintage per month' rule; documented as a limitation."})29# assembled-file checks30truth = pd.concat([old[old.date < new.date.min()].assign(vintage="2016-07-15"), new.assign(vintage="2026-10-28")])31m = asm.merge(truth, on="date", how="outer", suffixes=("_as", "_tr"), indicator=True)32ovl = m[(m.date >= ov.date.min()) & (m.date <= ov.date.max())]33if (ovl.vintage_as == "2016-07-15").any():34 issues.append({"issue": f"cpi_assembled.csv uses the older 2016-07-15 vintage for {int((ovl.vintage_as=='2016-07-15').sum())} overlap months instead of the latest vintage.",35 "handling": "Assembled file not used for estimation; levels taken directly from the vintage files."})36gaps = sorted(set(pd.date_range(asm.date.min(), asm.date.max(), freq="MS")) - set(asm.date))37tr_s = truth.set_index("date").cpi; as_s = asm.set_index("date").cpi38shifted = [d for d in tr_s.index if d >= pd.Timestamp("2016-07-01") and (d + pd.DateOffset(months=1)) in as_s.index39 and ((np.isclose(as_s[d + pd.DateOffset(months=1)], tr_s[d])) or (pd.isna(tr_s[d]) and pd.isna(as_s[d + pd.DateOffset(months=1)])))]40n_after = int((tr_s.index >= pd.Timestamp("2016-07-01")).sum())41if gaps or shifted:42 issues.append({"issue": f"cpi_assembled.csv omits month(s) {', '.join(g.strftime('%Y-%m') for g in gaps)} and {len(shifted)} of the {n_after} 2026-vintage values from 2016-07 onward appear one month late (series ends at a spurious 2026-10; the missing 2025-10 value appears as 2025-11).",43 "handling": "Assembled file not used for estimation; levels taken directly from the vintage files."})4445# --- merged series: latest vintage wins ---46s_old = old.set_index("date").cpi; s_new = new.set_index("date").cpi47idx = s_old.index.union(s_new.index)48P = pd.Series(np.nan, index=idx); V = pd.Series("", index=idx, dtype=object)49P[s_old.index] = s_old; V[s_old.index] = "2016-07-15"50P[s_new.index] = s_new; V[s_new.index] = "2026-10-28" # covers month even if value empty -> no fallback51n_merged = len(P)52win = pd.date_range(W0, W1, freq="MS")53lag = P.reindex(win - pd.DateOffset(months=12)).values54a = pd.DataFrame({"date": win, "cpi": P.reindex(win).values, "cpi_lag12": lag, "vintage_used": V.reindex(win).values})55a["growth_12m"] = 100 * (a.cpi / a.cpi_lag12 - 1)5657excl = []58def step(name, rule, before, after, why):59 excl.append(dict(step=name, rule=rule, n_before=before, n_after=after, n_removed=before - after, rationale=why))60step(1, "merge vintages (latest vintage per month) and restrict to window 2001-01..2026-09", n_merged, len(a),61 "months 1947-01..2000-12 outside the summary window (still used as 12-month lags)")62b1 = a[a.cpi.notna()]63step(2, "drop months with no index level P_t in the latest covering vintage", len(a), len(b1), "no interpolation/imputation allowed")64b2 = b1[b1.cpi_lag12.notna()]65step(3, "drop months with no index level P_{t-12}", len(b1), len(b2), "no interpolation/imputation allowed")66an = b2.copy()6768g = an.set_index("date").growth_12m69def gm(d):70 t = pd.Timestamp(d + "-01"); return round(float(g[t]), 10) if t in g.index else None71sub1 = g[(g.index >= "2001-01-01") & (g.index <= "2013-12-01")]72sub2 = g[(g.index >= "2014-01-01") & (g.index <= "2026-09-01")]73n_missing = len(a) - len(an)74lastm = P[P.notna()].index.max()75res = {76 "task_id": "F1-conf-01",77 "first_growth_month": g.index.min().strftime("%Y-%m"),78 "last_month": lastm.strftime("%Y-%m"),79 "n_months_growth": int(len(an)),80 "n_months_missing_growth": int(n_missing),81 "n_analysed": int(len(an)),82 "mean_growth_12m": round(float(g.mean()), 10),83 "mean_growth_12m_2001_2013": round(float(sub1.mean()), 10),84 "mean_growth_12m_2014_2026": round(float(sub2.mean()), 10),85 "growth_2014_06": gm("2014-06"), "growth_2016_07": gm("2016-07"), "growth_2020_05": gm("2020-05"),86 "growth_2022_06": gm("2022-06"), "growth_2026_09": gm("2026-09"),87 "max_growth_12m": round(float(g.max()), 10), "max_growth_month": g.idxmax().strftime("%Y-%m"),88 "min_growth_12m": round(float(g.min()), 10), "min_growth_month": g.idxmin().strftime("%Y-%m"),89 "feasibility": "complete" if n_missing == 0 else "partial",90 "feasibility_reason": None if n_missing == 0 else91 f"g_t not computable for {n_missing} window month(s) ({', '.join(a[a.growth_12m.isna()].date.dt.strftime('%Y-%m'))}) because the source publishes no index level; means/extremes are over the {len(an)} computable months (2014-2026 sub-period: {len(sub2)} of {len(a[a.date>='2014-01-01'])} months).",92 "data_issues": issues,93 "n_input_rows": n_rows,94 "random_seed": None,95 "n_months_window": int(len(a)),96 "n_months_growth_2001_2013": int(len(sub1)),97 "n_months_growth_2014_2026": int(len(sub2)),98}99os.makedirs(O, exist_ok=True)100with open(os.path.join(O, "results.json"), "w") as f: json.dump(res, f, indent=2)101out = an.copy(); out["date"] = out.date.dt.strftime("%Y-%m-01")102out.to_csv(os.path.join(O, "analytic.csv"), index=False, float_format="%.10g")103pd.DataFrame(excl).to_csv(os.path.join(O, "exclusions.csv"), index=False)104print(json.dumps({k: v for k, v in res.items() if k != "data_issues"}, indent=1))105