1"""F1-conf-01: 12-month CPI inflation from two ALFRED vintages.2Reads only data/; writes out/results.json, out/analytic.csv, out/exclusions.csv.3Deterministic; no randomness used (random_seed = null)."""4import json, os5import numpy as np6import pandas as pd78ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))9D = os.path.join(ROOT, "data"); O = os.path.join(ROOT, "out")10F_OLD, F_NEW, F_ASM = "cpi_vintage_2016-07-15.csv", "cpi_vintage_2026-10-28.csv", "cpi_assembled.csv"11V_OLD, V_NEW = "2016-07-15", "2026-10-28"12W0, W1 = pd.Timestamp("2001-01-01"), pd.Timestamp("2026-09-01")1314def load(f):15 d = pd.read_csv(os.path.join(D, f), dtype={"cpi": float})16 d["date"] = pd.to_datetime(d["date"])17 return d1819old, new, asm = load(F_OLD), load(F_NEW), load(F_ASM)20for d in (old, new):21 assert not d["date"].duplicated().any()22 assert d["date"].is_monotonic_increasing23issues = []2425# ---- Splice: most recent vintage wins wherever it covers the month (row present) ----26stack = pd.concat([old.assign(vintage=V_OLD), new.assign(vintage=V_NEW)], ignore_index=True)27n_stack = len(stack)28covered_new = set(new["date"])29spliced = stack[~((stack["vintage"] == V_OLD) & stack["date"].isin(covered_new))]30spliced = spliced.sort_values("date").set_index("date")31n_spliced = len(spliced)32full_idx = pd.date_range(spliced.index.min(), spliced.index.max(), freq="MS")33assert len(full_idx) == n_spliced, "gap in calendar after splice"3435missing_lvls = spliced.index[spliced["cpi"].isna()]36for m in missing_lvls:37 issues.append({"issue": f"Vintage {spliced.loc[m,'vintage']} reports no CPI value for {m:%Y-%m} (empty cell).",38 "handling": "Not imputed; g_t for that month (and for month+12 if inside the window) reported missing and counted."})3940# ---- Growth rates (no interpolation) ----41s = spliced.reindex(full_idx)42lag = s["cpi"].shift(12) # calendar-exact lag (index has no gaps)43g = 100.0 * (s["cpi"] / lag - 1.0)44win = s.loc[W0:W1].copy()45win["cpi_lag12"] = lag.loc[W0:W1]; win["growth_12m"] = g.loc[W0:W1]46n_window = len(win)47ok = win["growth_12m"].notna()48an = win[ok]49n_an = len(an)5051# cross-vintage revision size in the overlap (diagnostic for the splice)52ov = pd.merge(old, new, on="date", suffixes=("_old", "_new")).dropna()53rev = (ov["cpi_new"] - ov["cpi_old"])54issues.append({"issue": f"The two vintages overlap for {len(ov)} months ({ov.date.min():%Y-%m}..{ov.date.max():%Y-%m}) with different values "55 f"(revisions of seasonal factors; new-old ranges {rev.min():.3f}..{rev.max():.3f} index points).",56 "handling": "Per the availability rule the 2026-10-28 value is used for 2014-01 onward; 2016-07-15 values only before 2014-01. "57 "Hence g_t for 2014-01..2014-12 divides a 2026-vintage level by a 2016-vintage level (mixed-vintage ratio); noted as a limitation."})5859# ---- Audit of the hand-assembled file (not used for estimation) ----60ref = s[["cpi", "vintage"]].reset_index().rename(columns={"index": "date"})61c = pd.merge(asm, ref, on="date", how="outer", suffixes=("_asm", "_ref"), indicator=True)62only_asm = c[c["_merge"] == "left_only"]["date"]63only_ref = c[c["_merge"] == "right_only"]["date"]64both = c[c["_merge"] == "both"]65valdiff = both[~(np.isclose(both.cpi_asm, both.cpi_ref, atol=1e-9) | (both.cpi_asm.isna() & both.cpi_ref.isna()))]66oldused = both[(both.vintage_asm.astype(str) == V_OLD) & (both.vintage_ref == V_NEW)]67# detect one-month shift: asm value at t equals reference value at t-168sh = pd.merge(asm, ref.assign(date=ref["date"] + pd.offsets.MonthBegin(1)), on="date", suffixes=("_asm", "_prev"))69sh = sh[sh.date > pd.Timestamp("2016-07-01")]70n_shift = int((np.isclose(sh.cpi_asm, sh.cpi_prev, atol=1e-9) | (sh.cpi_asm.isna() & sh.cpi_prev.isna())).sum())71issues.append({"issue": f"cpi_assembled.csv uses 2016-07-15 vintage values for {len(oldused)} months "72 f"({oldused.date.min():%Y-%m}..{oldused.date.max():%Y-%m}) that the 2026-10-28 vintage covers, violating the most-recent-vintage rule.",73 "handling": "Assembled file not used for estimation; series re-built from the two vintage files."})74issues.append({"issue": f"cpi_assembled.csv is missing month(s) {', '.join(f'{d:%Y-%m}' for d in only_ref)} and contains month(s) "75 f"{', '.join(f'{d:%Y-%m}' for d in only_asm)} not present in any source; from 2016-08 onward {n_shift} of {len(sh)} values "76 f"equal the source value of the previous month (one-month date shift). In total {len(valdiff)} common months have values differing from the authoritative splice.",77 "handling": "Assembled file treated as unreliable and used only for this audit; all estimates come from the vintage files."})78issues.append({"issue": "SHA-256 values in data/README.md refer to the raw downloads, not to the reformatted CSV files in data/, so they cannot be used to verify the files.",79 "handling": "Recorded the SHA-256 of the files as received in out/state/data_audit.md; no further action."})8081# ---- Outputs ----82ym = lambda t: f"{t:%Y-%m}"83def gm(t):84 t = pd.Timestamp(t); v = win.loc[t, "growth_12m"]85 return None if pd.isna(v) else float(v)86gg = an["growth_12m"]87so = old.set_index("date")["cpi"]88g_old = 100.0 * (so / so.shift(12) - 1.0)89missing_months = [ym(t) for t in win.index[~ok]]90res = {91 "task_id": "F1-conf-01",92 "first_growth_month": ym(an.index.min()),93 "last_month": ym(s.index[s["cpi"].notna() & (s.index <= W1)].max()),94 "n_months_growth": int(n_an),95 "n_months_missing_growth": int(n_window - n_an),96 "n_analysed": int(n_an),97 "mean_growth_12m": float(gg.mean()),98 "mean_growth_12m_2001_2013": float(gg.loc["2001-01-01":"2013-12-01"].mean()),99 "mean_growth_12m_2014_2026": float(gg.loc["2014-01-01":"2026-09-01"].mean()),100 "growth_2014_06": gm("2014-06-01"), "growth_2016_07": gm("2016-07-01"),101 "growth_2020_05": gm("2020-05-01"), "growth_2022_06": gm("2022-06-01"),102 "growth_2026_09": gm("2026-09-01"),103 "max_growth_12m": float(gg.max()), "max_growth_month": ym(gg.idxmax()),104 "min_growth_12m": float(gg.min()), "min_growth_month": ym(gg.idxmin()),105 "feasibility": "complete" if n_an == n_window else "partial",106 "feasibility_reason": None if n_an == n_window else107 (f"{n_window - n_an} of {n_window} window months have no computable 12-month growth rate ({', '.join(missing_months)}) because the "108 "source reports no index level; per the task rules they are not imputed. All window means are over the remaining months; all named months are computable."),109 "data_issues": issues,110 "n_input_rows": {F_OLD: int(len(old)), F_NEW: int(len(new)), F_ASM: int(len(asm))},111 "random_seed": None,112 # auxiliary (not required) keys113 "n_months_window": int(n_window),114 "missing_growth_months": missing_months,115 "n_months_growth_2001_2013": int(gg.loc["2001-01-01":"2013-12-01"].size),116 "n_months_growth_2014_2026": int(gg.loc["2014-01-01":"2026-09-01"].size),117 "sd_growth_12m": float(gg.std(ddof=1)),118 "max_abs_vintage_revision_overlap": float(rev.abs().max()),119 # sensitivity (not an estimand): g_t for 2014-01..2016-06 computed from the 2016-07-15 vintage alone120 "sens_mean_growth_201401_201606_old_vintage": float(g_old.loc["2014-01-01":"2016-06-01"].mean()),121 "sens_mean_growth_201401_201606_rule": float(gg.loc["2014-01-01":"2016-06-01"].mean()),122 "sens_max_abs_diff_growth_201401_201606": float((g_old.loc["2014-01-01":"2016-06-01"] - gg.loc["2014-01-01":"2016-06-01"]).abs().max()),123}124out = an.reset_index().rename(columns={"index": "date"})125out["date"] = out["date"].dt.strftime("%Y-%m-01")126out = out.rename(columns={"vintage": "vintage_used"})[["date", "cpi", "cpi_lag12", "vintage_used", "growth_12m"]]127out.to_csv(os.path.join(O, "analytic.csv"), index=False, float_format="%.10g")128129n_before_win = n_spliced130ex = [131 ("1", "Drop 2016-07-15-vintage rows for months covered by the 2026-10-28 vintage", n_stack, n_spliced,132 "Information rule: use the most recent vintage covering the month"),133 ("2", f"Keep months in window {ym(W0)}..{ym(W1)}", n_spliced, n_window,134 "Earlier months only serve as 12-month lags"),135 ("3", "Drop window months without a computable g_t (P_t or P_t-12 missing)", n_window, n_an,136 "No interpolation/imputation allowed; missing: " + ";".join(missing_months)),137]138exd = pd.DataFrame(ex, columns=["step", "rule", "n_before", "n_after", "rationale"])139exd["n_removed"] = exd.n_before - exd.n_after140exd = exd[["step", "rule", "n_before", "n_after", "n_removed", "rationale"]]141exd.to_csv(os.path.join(O, "exclusions.csv"), index=False)142143with open(os.path.join(O, "results.json"), "w") as f:144 json.dump(res, f, indent=2)145print(f"n_analysed={n_an} of {n_window}; mean={res['mean_growth_12m']:.4f}; "146 f"2001-13={res['mean_growth_12m_2001_2013']:.4f}; 2014-26={res['mean_growth_12m_2014_2026']:.4f}")147