1"""F1-conf-01: 12-month CPI inflation from two ALFRED vintages.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Reads only data/; writes out/results.json, out/analytic.csv, out/exclusions.csv. No randomness."""4import json, os5import numpy as np6import pandas as pd78D, O = "data", "out"9OLD, NEW, ASM = "cpi_vintage_2016-07-15.csv", "cpi_vintage_2026-10-28.csv", "cpi_assembled.csv"10W0, W1 = pd.Timestamp("2001-01-01"), pd.Timestamp("2026-09-01")1112old = pd.read_csv(os.path.join(D, OLD), parse_dates=["date"])13new = pd.read_csv(os.path.join(D, NEW), parse_dates=["date"])14asm = pd.read_csv(os.path.join(D, ASM), parse_dates=["date"]) # read for audit only15old["vintage_used"], new["vintage_used"] = "2016-07-15", "2026-10-28"1617# --- audit of the hand-assembled file against the rule-based splice (not used for computation)18issues = []19ledger = []20def step(name, rule, nb, na, why):21 ledger.append(dict(step=name, rule=rule, n_before=nb, n_after=na, n_removed=nb - na, rationale=why))2223# --- splice: latest vintage wins for every month it covers (row present, even if value empty)24n_read = len(old) + len(new)25superseded = old["date"].isin(new["date"])26series = pd.concat([old[~superseded], new], ignore_index=True).sort_values("date").reset_index(drop=True)27step("1_splice", "drop 2016-07-15 vintage rows for months covered by the 2026-10-28 vintage",28 n_read, len(series), "information rule: use most recent vintage in which the month is available")29assert series["date"].is_unique30full = pd.date_range(series["date"].min(), series["date"].max(), freq="MS")31assert len(full) == len(series), "gaps in month sequence"3233s = series.set_index("date")34lag = s["cpi"].shift(12) # contiguous monthly index verified above -> shift(12) = t-12 months35lag_v = s["vintage_used"].shift(12)36s["cpi_lag12"] = lag37s["growth_12m"] = 100.0 * (s["cpi"] / s["cpi_lag12"] - 1.0) # NaN if either level missing: no imputation3839win = s.loc[W0:W1].copy()40step("2_window", "keep months 2001-01 .. 2026-09", len(series), len(win), "estimation window defined in TASK.md")41missing_months = [d.strftime("%Y-%m") for d in win.index[win["growth_12m"].isna()]]42ana = win[win["growth_12m"].notna()].copy()43step("3_computable", "drop months where P_t or P_{t-12} is missing in the sources",44 len(win), len(ana), "no interpolation/imputation allowed; missing months: " + ";".join(missing_months))4546# --- data issues47miss_lvl = new.loc[new["cpi"].isna(), "date"].dt.strftime("%Y-%m").tolist()48if miss_lvl:49 issues.append({"issue": f"2026-10-28 vintage reports no value (empty cpi) for {', '.join(miss_lvl)}",50 "handling": "not imputed; g_t for affected window months (" + ", ".join(missing_months)51 + ") reported missing and excluded from means; the latest vintage still governs that month, so the older vintage is not used as a fallback (it does not cover it anyway)"})52ov = old.merge(new, on="date", suffixes=("_old", "_new"))53d = (ov["cpi_old"] - ov["cpi_new"]).abs()54issues.append({"issue": f"vintages overlap for {len(ov)} months ({ov.date.min():%Y-%m}..{ov.date.max():%Y-%m}) with different values (mean abs diff {d.mean():.3f}, max {d.max():.3f} index points; seasonal-factor revisions)",55 "handling": "2026-10-28 vintage values used for overlap months per information rule"})56mixed = ana[ana["vintage_used"] != lag_v.reindex(ana.index)]57if len(mixed):58 issues.append({"issue": f"for {len(mixed)} months ({mixed.index.min():%Y-%m}..{mixed.index.max():%Y-%m}) P_t comes from the 2026-10-28 vintage but P_(t-12) only exists in the 2016-07-15 vintage (new extract starts 2014-01), so the ratio mixes vintages",59 "handling": "computed as prescribed by the latest-vintage rule; flagged as a limitation (revision noise in those 12 growth rates)"})60# assembled-file checks61ref = series[["date", "cpi", "vintage_used"]].merge(asm, on="date", how="outer", indicator=True)62only_asm = ref.loc[ref["_merge"] == "right_only", "date"].dt.strftime("%Y-%m").tolist()63only_ref = ref.loc[ref["_merge"] == "left_only", "date"].dt.strftime("%Y-%m").tolist()64both = ref[ref["_merge"] == "both"]65val_bad = both[~((both["cpi_x"] == both["cpi_y"]) | (both["cpi_x"].isna() & both["cpi_y"].isna()))]66if only_asm or only_ref or len(val_bad):67 issues.append({"issue": f"cpi_assembled.csv contains assembly errors: months missing {only_ref}, extra months {only_asm}, "68 f"{len(val_bad)} months whose value differs from the rule-based splice (2014-01..2016-06 taken from the old vintage; "69 "from 2016-08 on, each new-vintage value is labelled one month late)",70 "handling": "assembled file not used for computation; series rebuilt from the two authoritative vintage files"})7172def g(month):73 t = pd.Timestamp(month + "-01")74 v = s["growth_12m"].get(t, np.nan)75 return None if pd.isna(v) else round(float(v), 10)7677gr = ana["growth_12m"]78last_level = s.loc[:W1, "cpi"].dropna().index.max()79feas = "complete" if not missing_months else "partial"80res = {81 "task_id": "F1-conf-01",82 "first_growth_month": ana.index.min().strftime("%Y-%m"),83 "last_month": last_level.strftime("%Y-%m"),84 "n_months_growth": int(len(ana)),85 "n_months_missing_growth": int(len(missing_months)),86 "n_analysed": int(len(ana)),87 "mean_growth_12m": round(float(gr.mean()), 10),88 "mean_growth_12m_2001_2013": round(float(gr.loc["2001-01-01":"2013-12-01"].mean()), 10),89 "mean_growth_12m_2014_2026": round(float(gr.loc["2014-01-01":"2026-09-01"].mean()), 10),90 "growth_2014_06": g("2014-06"), "growth_2016_07": g("2016-07"), "growth_2020_05": g("2020-05"),91 "growth_2022_06": g("2022-06"), "growth_2026_09": g("2026-09"),92 "max_growth_12m": round(float(gr.max()), 10), "max_growth_month": gr.idxmax().strftime("%Y-%m"),93 "min_growth_12m": round(float(gr.min()), 10), "min_growth_month": gr.idxmin().strftime("%Y-%m"),94 "feasibility": feas,95 "feasibility_reason": None if feas == "complete" else96 f"g_t not computable for {len(missing_months)} window month(s) ({', '.join(missing_months)}) because the source publishes no index level for {', '.join(miss_lvl)}; all summary estimands computed over the remaining {len(ana)} months",97 "data_issues": issues,98 "n_input_rows": {OLD: int(len(old)), NEW: int(len(new)), ASM: int(len(asm))},99 "random_seed": None,100 # auxiliary diagnostics101 "n_months_window": int(len(win)),102 "n_months_2001_2013": int(gr.loc["2001-01-01":"2013-12-01"].size),103 "n_months_2014_2026": int(gr.loc["2014-01-01":"2026-09-01"].size),104 "missing_growth_months": missing_months,105 "sd_growth_12m": round(float(gr.std(ddof=1)), 10),106}107os.makedirs(O, exist_ok=True)108out = ana.reset_index()[["date", "cpi", "cpi_lag12", "vintage_used", "growth_12m"]]109out["date"] = out["date"].dt.strftime("%Y-%m-%d")110out.to_csv(os.path.join(O, "analytic.csv"), index=False, float_format="%.10f")111pd.DataFrame(ledger).to_csv(os.path.join(O, "exclusions.csv"), index=False)112with open(os.path.join(O, "results.json"), "w") as f:113 json.dump(res, f, indent=2)114print(json.dumps({k: v for k, v in res.items() if k != "data_issues"}, indent=1))115