"""F1-conf-01: 12-month CPIAUCSL growth from two ALFRED vintages.
Run from workspace root: ~/wp29/venv/bin/python out/analysis.py
Reads only data/; writes out/results.json, out/analytic.csv, out/exclusions.csv. No randomness."""
import json, os
import numpy as np, pandas as pd

D, O = "data", "out"
OLD, NEW, ASM = "cpi_vintage_2016-07-15.csv", "cpi_vintage_2026-10-28.csv", "cpi_assembled.csv"
W0, W1 = pd.Timestamp("2001-01-01"), pd.Timestamp("2026-09-01")

old = pd.read_csv(os.path.join(D, OLD), parse_dates=["date"])
new = pd.read_csv(os.path.join(D, NEW), parse_dates=["date"])
asm = pd.read_csv(os.path.join(D, ASM), parse_dates=["date"], dtype={"vintage": str})
n_input_rows = {OLD: len(old), NEW: len(new), ASM: len(asm)}

for name, d in [("old", old), ("new", new)]:
    assert not d.date.duplicated().any(), name
    assert d.date.is_monotonic_increasing, name

# ---- Splice: most recent vintage covering the month. "Covering" = month is a row of the
# vintage extract. A row present with an empty cpi means the source reports no value; we do NOT
# fall back to the older vintage (it does not cover such months anyway here) and do not impute.
new_cov = set(new.date)
lv = pd.concat([old[~old.date.isin(new_cov)].assign(vintage_used="2016-07-15"),
                new.assign(vintage_used="2026-10-28")]).sort_values("date").reset_index(drop=True)
full = pd.date_range(lv.date.min(), lv.date.max(), freq="MS")
assert len(full) == len(lv) and (lv.date.values == full.values).all()
s = lv.set_index("date")

# ---- Audit of the hand-assembled file against the authoritative splice
issues = []
m = asm.merge(lv, on="date", how="outer", suffixes=("_asm", "_ref"), indicator=True)
val_diff = (m._merge == "both") & ~((m.cpi_asm - m.cpi_ref).abs() < 1e-9) & ~(m.cpi_asm.isna() & m.cpi_ref.isna())
missing_in_asm = m.loc[m._merge == "right_only", "date"].dt.strftime("%Y-%m").tolist()
extra_in_asm = m.loc[m._merge == "left_only", "date"].dt.strftime("%Y-%m").tolist()
ov = (m.date >= "2014-01-01") & (m.date <= "2016-06-01")
n_ov_oldv = int((ov & (m.vintage == "2016-07-15")).sum())
if n_ov_oldv:
    issues.append({"issue": f"cpi_assembled.csv uses the 2016-07-15 vintage for {n_ov_oldv} overlap months 2014-01..2016-06 that the 2026-10-28 vintage covers (violates the most-recent-vintage rule).",
                   "handling": "Assembled file not used for computation; levels taken directly from the two vintage files."})
if missing_in_asm or extra_in_asm:
    # check: assembled value at month t+1 equals true value at month t for months after the gap
    a_ser = asm.set_index("date").cpi
    post = s.loc["2016-07-01":].cpi
    a_next = a_ser.reindex(post.index + pd.DateOffset(months=1))
    both = ~(post.isna().values | a_next.isna().values)
    shifted = bool(np.allclose(post.values[both], a_next.values[both]))
    issues.append({"issue": f"cpi_assembled.csv omits month(s) {missing_in_asm} and contains month(s) {extra_in_asm} absent from the sources; every value from 2016-07 on is stamped one month later than its true date (one-month shift confirmed: {shifted}); e.g. its 2026-10 value is the 2026-09 level and its 2025-11 cell is the empty 2025-10 cell.",
                   "handling": "Assembled file not used for computation; it is read only for this audit."})
issues.append({"issue": f"cpi_assembled.csv: {int(val_diff.sum())} common-date rows disagree in value with the authoritative splice of the two vintage files.",
               "handling": "Vintage files treated as authoritative per TASK.md."})
miss_lv = s.index[s.cpi.isna()].strftime("%Y-%m").tolist()
if miss_lv:
    issues.append({"issue": f"The 2026-10-28 vintage reports no index level for {miss_lv} (empty cell).",
                   "handling": "Not imputed or interpolated and no fallback to an older vintage; g_t is missing for that month. The month 12 later (2026-10) would also be affected but lies outside the window."})
ovd = old.merge(new, on="date", suffixes=("_old", "_new")).dropna()
dd = ovd.cpi_new - ovd.cpi_old
issues.append({"issue": f"The two vintages overlap on {len(ovd)} months (2014-01..2016-06) with revised levels (new minus old: min {dd.min():.3f}, max {dd.max():.3f} index points), consistent with seasonal-factor revisions.",
               "handling": "Per the rule, the 2026-10-28 vintage is used for all overlap months; consequently g_t for 2014-01..2014-12 is a ratio of a 2026-vintage level to a 2016-vintage level (cross-vintage splice), reported as is."})
issues.append({"issue": "SHA-256 values listed in data/README.md refer to the raw downloads before reformatting and do not match the provided CSV files.",
               "handling": "Hashes of the provided files recorded in out/state/data_audit.md; README states reformatting did not change values. Not used to reject data."})

# ---- Growth
s["cpi_lag12"] = s.cpi.shift(12)  # complete monthly index, so shift(12) == month t-12
s["growth_12m"] = 100 * (s.cpi / s.cpi_lag12 - 1)
win = s.loc[W0:W1].copy()
n_window = len(win)
assert n_window == len(pd.date_range(W0, W1, freq="MS"))
ok = win.growth_12m.notna()
an = win[ok].copy()
missing_g = win.index[~ok].strftime("%Y-%m").tolist()

ex = []
def step(name, rule, nb, na, why):
    ex.append(dict(step=name, rule=rule, n_before=nb, n_after=na, n_removed=nb - na, rationale=why))
step("1_restrict_window", "keep months 2001-01..2026-09 of the spliced monthly level series (1947-01..2026-09)", len(s), n_window,
     "Analysis window defined in TASK.md")
n_curnan = int(win.cpi.isna().sum())
step("2_drop_missing_level_t", "drop months whose own index level P_t is missing in the source", n_window, n_window - n_curnan,
     "No imputation allowed (empty cell in 2026-10-28 vintage)")
step("3_drop_missing_lag12", "drop months whose P_(t-12) is missing in the source", n_window - n_curnan, len(an),
     "No imputation allowed")

an_out = an.reset_index()[["date", "cpi", "cpi_lag12", "vintage_used", "growth_12m"]]
an_out["date"] = an_out.date.dt.strftime("%Y-%m-01")
an_out.to_csv(os.path.join(O, "analytic.csv"), index=False, float_format="%.10g")
pd.DataFrame(ex).to_csv(os.path.join(O, "exclusions.csv"), index=False)

g = an.growth_12m
def gm(ym):
    v = win.growth_12m.get(pd.Timestamp(ym + "-01"), np.nan)
    return None if pd.isna(v) else round(float(v), 6)
r6 = lambda x: round(float(x), 6)
last_level = s.index[s.cpi.notna() & (s.index <= W1)].max()
res = {
    "task_id": "F1-conf-01",
    "first_growth_month": g.index.min().strftime("%Y-%m"),
    "last_month": last_level.strftime("%Y-%m"),
    "n_months_growth": int(len(g)),
    "n_months_missing_growth": int(n_window - len(g)),
    "missing_growth_months": missing_g,
    "n_analysed": int(len(g)),
    "mean_growth_12m": r6(g.mean()),
    "mean_growth_12m_2001_2013": r6(g.loc["2001-01-01":"2013-12-01"].mean()),
    "mean_growth_12m_2014_2026": r6(g.loc["2014-01-01":"2026-09-01"].mean()),
    "n_growth_2001_2013": int(g.loc["2001-01-01":"2013-12-01"].size),
    "n_growth_2014_2026": int(g.loc["2014-01-01":"2026-09-01"].size),
    "growth_2014_06": gm("2014-06"), "growth_2016_07": gm("2016-07"), "growth_2020_05": gm("2020-05"),
    "growth_2022_06": gm("2022-06"), "growth_2026_09": gm("2026-09"),
    "max_growth_12m": r6(g.max()), "max_growth_month": g.idxmax().strftime("%Y-%m"),
    "min_growth_12m": r6(g.min()), "min_growth_month": g.idxmin().strftime("%Y-%m"),
    "feasibility": "complete" if not missing_g else "partial",
    "feasibility_reason": None if not missing_g else
        f"g_t not computable for {missing_g}: the 2026-10-28 vintage reports no index level for {miss_lv}; no imputation per TASK.md. All summary statistics use the {len(g)} computable months.",
    "data_issues": issues,
    "n_input_rows": n_input_rows,
    "random_seed": None,
}
with open(os.path.join(O, "results.json"), "w") as f:
    json.dump(res, f, indent=2)
print(json.dumps({k: v for k, v in res.items() if k != "data_issues"}, indent=1))