1"""F1-conf-01: 12-month CPIAUCSL growth from two ALFRED vintages.2Run from workspace root: ~/wp29/venv/bin/python out/analysis.py3Reads only data/; writes out/results.json, out/analytic.csv, out/exclusions.csv. No randomness."""4import json, os5import numpy as np, pandas as pd67D, O = "data", "out"8OLD, NEW, ASM = "cpi_vintage_2016-07-15.csv", "cpi_vintage_2026-10-28.csv", "cpi_assembled.csv"9W0, W1 = pd.Timestamp("2001-01-01"), pd.Timestamp("2026-09-01")1011old = pd.read_csv(os.path.join(D, OLD), parse_dates=["date"])12new = pd.read_csv(os.path.join(D, NEW), parse_dates=["date"])13asm = pd.read_csv(os.path.join(D, ASM), parse_dates=["date"], dtype={"vintage": str})14n_input_rows = {OLD: len(old), NEW: len(new), ASM: len(asm)}1516for name, d in [("old", old), ("new", new)]:17 assert not d.date.duplicated().any(), name18 assert d.date.is_monotonic_increasing, name1920# ---- Splice: most recent vintage covering the month. "Covering" = month is a row of the21# vintage extract. A row present with an empty cpi means the source reports no value; we do NOT22# fall back to the older vintage (it does not cover such months anyway here) and do not impute.23new_cov = set(new.date)24lv = pd.concat([old[~old.date.isin(new_cov)].assign(vintage_used="2016-07-15"),25 new.assign(vintage_used="2026-10-28")]).sort_values("date").reset_index(drop=True)26full = pd.date_range(lv.date.min(), lv.date.max(), freq="MS")27assert len(full) == len(lv) and (lv.date.values == full.values).all()28s = lv.set_index("date")2930# ---- Audit of the hand-assembled file against the authoritative splice31issues = []32m = asm.merge(lv, on="date", how="outer", suffixes=("_asm", "_ref"), indicator=True)33val_diff = (m._merge == "both") & ~((m.cpi_asm - m.cpi_ref).abs() < 1e-9) & ~(m.cpi_asm.isna() & m.cpi_ref.isna())34missing_in_asm = m.loc[m._merge == "right_only", "date"].dt.strftime("%Y-%m").tolist()35extra_in_asm = m.loc[m._merge == "left_only", "date"].dt.strftime("%Y-%m").tolist()36ov = (m.date >= "2014-01-01") & (m.date <= "2016-06-01")37n_ov_oldv = int((ov & (m.vintage == "2016-07-15")).sum())38if n_ov_oldv:39 issues.append({"issue": f"cpi_assembled.csv uses the 2016-07-15 vintage for {n_ov_oldv} overlap months 2014-01..2016-06 that the 2026-10-28 vintage covers (violates the most-recent-vintage rule).",40 "handling": "Assembled file not used for computation; levels taken directly from the two vintage files."})41if missing_in_asm or extra_in_asm:42 # check: assembled value at month t+1 equals true value at month t for months after the gap43 a_ser = asm.set_index("date").cpi44 post = s.loc["2016-07-01":].cpi45 a_next = a_ser.reindex(post.index + pd.DateOffset(months=1))46 both = ~(post.isna().values | a_next.isna().values)47 shifted = bool(np.allclose(post.values[both], a_next.values[both]))48 issues.append({"issue": f"cpi_assembled.csv omits month(s) {missing_in_asm} and contains month(s) {extra_in_asm} absent from the sources; every value from 2016-07 on is stamped one month later than its true date (one-month shift confirmed: {shifted}); e.g. its 2026-10 value is the 2026-09 level and its 2025-11 cell is the empty 2025-10 cell.",49 "handling": "Assembled file not used for computation; it is read only for this audit."})50issues.append({"issue": f"cpi_assembled.csv: {int(val_diff.sum())} common-date rows disagree in value with the authoritative splice of the two vintage files.",51 "handling": "Vintage files treated as authoritative per TASK.md."})52miss_lv = s.index[s.cpi.isna()].strftime("%Y-%m").tolist()53if miss_lv:54 issues.append({"issue": f"The 2026-10-28 vintage reports no index level for {miss_lv} (empty cell).",55 "handling": "Not imputed or interpolated and no fallback to an older vintage; g_t is missing for that month. The month 12 later (2026-10) would also be affected but lies outside the window."})56ovd = old.merge(new, on="date", suffixes=("_old", "_new")).dropna()57dd = ovd.cpi_new - ovd.cpi_old58issues.append({"issue": f"The two vintages overlap on {len(ovd)} months (2014-01..2016-06) with revised levels (new minus old: min {dd.min():.3f}, max {dd.max():.3f} index points), consistent with seasonal-factor revisions.",59 "handling": "Per the rule, the 2026-10-28 vintage is used for all overlap months; consequently g_t for 2014-01..2014-12 is a ratio of a 2026-vintage level to a 2016-vintage level (cross-vintage splice), reported as is."})60issues.append({"issue": "SHA-256 values listed in data/README.md refer to the raw downloads before reformatting and do not match the provided CSV files.",61 "handling": "Hashes of the provided files recorded in out/state/data_audit.md; README states reformatting did not change values. Not used to reject data."})6263# ---- Growth64s["cpi_lag12"] = s.cpi.shift(12) # complete monthly index, so shift(12) == month t-1265s["growth_12m"] = 100 * (s.cpi / s.cpi_lag12 - 1)66win = s.loc[W0:W1].copy()67n_window = len(win)68assert n_window == len(pd.date_range(W0, W1, freq="MS"))69ok = win.growth_12m.notna()70an = win[ok].copy()71missing_g = win.index[~ok].strftime("%Y-%m").tolist()7273ex = []74def step(name, rule, nb, na, why):75 ex.append(dict(step=name, rule=rule, n_before=nb, n_after=na, n_removed=nb - na, rationale=why))76step("1_restrict_window", "keep months 2001-01..2026-09 of the spliced monthly level series (1947-01..2026-09)", len(s), n_window,77 "Analysis window defined in TASK.md")78n_curnan = int(win.cpi.isna().sum())79step("2_drop_missing_level_t", "drop months whose own index level P_t is missing in the source", n_window, n_window - n_curnan,80 "No imputation allowed (empty cell in 2026-10-28 vintage)")81step("3_drop_missing_lag12", "drop months whose P_(t-12) is missing in the source", n_window - n_curnan, len(an),82 "No imputation allowed")8384an_out = an.reset_index()[["date", "cpi", "cpi_lag12", "vintage_used", "growth_12m"]]85an_out["date"] = an_out.date.dt.strftime("%Y-%m-01")86an_out.to_csv(os.path.join(O, "analytic.csv"), index=False, float_format="%.10g")87pd.DataFrame(ex).to_csv(os.path.join(O, "exclusions.csv"), index=False)8889g = an.growth_12m90def gm(ym):91 v = win.growth_12m.get(pd.Timestamp(ym + "-01"), np.nan)92 return None if pd.isna(v) else round(float(v), 6)93r6 = lambda x: round(float(x), 6)94last_level = s.index[s.cpi.notna() & (s.index <= W1)].max()95res = {96 "task_id": "F1-conf-01",97 "first_growth_month": g.index.min().strftime("%Y-%m"),98 "last_month": last_level.strftime("%Y-%m"),99 "n_months_growth": int(len(g)),100 "n_months_missing_growth": int(n_window - len(g)),101 "missing_growth_months": missing_g,102 "n_analysed": int(len(g)),103 "mean_growth_12m": r6(g.mean()),104 "mean_growth_12m_2001_2013": r6(g.loc["2001-01-01":"2013-12-01"].mean()),105 "mean_growth_12m_2014_2026": r6(g.loc["2014-01-01":"2026-09-01"].mean()),106 "n_growth_2001_2013": int(g.loc["2001-01-01":"2013-12-01"].size),107 "n_growth_2014_2026": int(g.loc["2014-01-01":"2026-09-01"].size),108 "growth_2014_06": gm("2014-06"), "growth_2016_07": gm("2016-07"), "growth_2020_05": gm("2020-05"),109 "growth_2022_06": gm("2022-06"), "growth_2026_09": gm("2026-09"),110 "max_growth_12m": r6(g.max()), "max_growth_month": g.idxmax().strftime("%Y-%m"),111 "min_growth_12m": r6(g.min()), "min_growth_month": g.idxmin().strftime("%Y-%m"),112 "feasibility": "complete" if not missing_g else "partial",113 "feasibility_reason": None if not missing_g else114 f"g_t not computable for {missing_g}: the 2026-10-28 vintage reports no index level for {miss_lv}; no imputation per TASK.md. All summary statistics use the {len(g)} computable months.",115 "data_issues": issues,116 "n_input_rows": n_input_rows,117 "random_seed": None,118}119with open(os.path.join(O, "results.json"), "w") as f:120 json.dump(res, f, indent=2)121print(json.dumps({k: v for k, v in res.items() if k != "data_issues"}, indent=1))122