"""F1-conf-01: 12-month CPI inflation from two ALFRED vintages.
Run from workspace root: ~/wp29/venv/bin/python out/analysis.py
Reads only data/; writes out/results.json, out/analytic.csv, out/exclusions.csv. No randomness."""
import json
import numpy as np
import pandas as pd

D = "data/"
F_OLD, F_NEW, F_ASM = "cpi_vintage_2016-07-15.csv", "cpi_vintage_2026-10-28.csv", "cpi_assembled.csv"
W0, W1 = pd.Timestamp("2001-01-01"), pd.Timestamp("2026-09-01")

old = pd.read_csv(D + F_OLD, parse_dates=["date"])
new = pd.read_csv(D + F_NEW, parse_dates=["date"])
asm = pd.read_csv(D + F_ASM, parse_dates=["date"], dtype={"vintage": str})
n_input_rows = {F_OLD: len(old), F_NEW: len(new), F_ASM: len(asm)}

issues = []
for name, df in [(F_OLD, old), (F_NEW, new)]:
    assert not df.date.duplicated().any(), name
    full = pd.date_range(df.date.min(), df.date.max(), freq="MS")
    assert len(full) == len(df), f"gap in {name}"
miss_new = new.loc[new.cpi.isna(), "date"].dt.strftime("%Y-%m").tolist()
if miss_new:
    issues.append({"issue": f"{F_NEW} has no index value (empty cpi) for {', '.join(miss_new)}",
                   "handling": "Not imputed or back-filled from another source; growth for that month (and for the month 12 later) is reported as missing and counted."})

# Overlap / revisions
ov = old.merge(new, on="date", suffixes=("_old", "_new"))
ndiff = int((ov.cpi_old != ov.cpi_new).sum())
if ndiff:
    issues.append({"issue": f"The two vintages overlap for {len(ov)} months ({ov.date.min():%Y-%m} to {ov.date.max():%Y-%m}) and differ in {ndiff} of them (revisions; max abs diff {np.abs(ov.cpi_old-ov.cpi_new).max():.3f} index points)",
                   "handling": "Per the availability rule, the 2026-10-28 vintage value is used for every month it covers; 2016-07-15 values only before 2014-01. Growth rates for 2014-01..2014-12 therefore combine a 2026-vintage numerator with a 2016-vintage base."})

# Most-recent-vintage series
lv = pd.concat([old.assign(vintage_used="2016-07-15"), new.assign(vintage_used="2026-10-28")])
lv = lv.sort_values(["date", "vintage_used"]).drop_duplicates("date", keep="last").set_index("date")
months = pd.date_range(lv.index.min(), lv.index.max(), freq="MS")
lv = lv.reindex(months)
assert lv.vintage_used.notna().all()

# Audit assembled file against authoritative series
a = asm.set_index("date")
asm_missing_months = [m.strftime("%Y-%m") for m in months if m not in a.index]
asm_extra = [m.strftime("%Y-%m") for m in a.index if m not in lv.index]
cmp = a.join(lv[["cpi", "vintage_used"]], rsuffix="_auth", how="inner")
mism = cmp[~((cmp.cpi == cmp.cpi_auth) | (cmp.cpi.isna() & cmp.cpi_auth.isna()))]
oldlab = cmp[(cmp.index >= new.date.min()) & (cmp.vintage != "2026-10-28")]
if len(mism) or asm_missing_months or asm_extra:
    issues.append({"issue": (f"{F_ASM} contains assembly errors: months absent {asm_missing_months}; months beyond the sources {asm_extra}; "
                             f"{len(mism)} months whose value differs from the most-recent-vintage value ({len(oldlab)} months 2014-01.. taken from the 2016 vintage although the 2026 vintage covers them; "
                             f"values after the missing month shifted one month later)"),
                   "handling": "File read only for auditing; all computations use the two authoritative vintage files directly."})
issues.append({"issue": "README SHA-256 values refer to the raw downloads before reformatting, so they cannot be matched against the provided files",
               "handling": "Recorded SHA-256 of provided files in out/state/data_audit.md; values not otherwise verifiable offline."})

# Growth
lv["cpi_lag12"] = lv.cpi.shift(12)
lv["growth_12m"] = 100 * (lv.cpi / lv.cpi_lag12 - 1)
win = lv.loc[W0:W1].copy()
n_window = len(win)
ok = win[win.growth_12m.notna()].copy()
missing_g = win.index[win.growth_12m.isna()].strftime("%Y-%m").tolist()

# Exclusion ledger
n_all = len(lv)
rows = [
    ("1", "Combined most-recent-vintage monthly series 1947-01..2026-09 restricted to window 2001-01..2026-09", n_all, n_window,
     "Window defined by the task"),
    ("2", "Drop window months without computable g_t (P_t or P_{t-12} missing in sources)", n_window, len(ok),
     f"No interpolation/imputation allowed; missing: {', '.join(missing_g) if missing_g else 'none'}"),
]
excl = pd.DataFrame([{"step": s, "rule": r, "n_before": b, "n_after": a_, "n_removed": b - a_, "rationale": ra} for s, r, b, a_, ra in rows])
excl.to_csv("out/exclusions.csv", index=False)

out = ok.reset_index().rename(columns={"index": "date"})
out["date"] = out.date.dt.strftime("%Y-%m-%d")
out[["date", "cpi", "cpi_lag12", "vintage_used", "growth_12m"]].to_csv("out/analytic.csv", index=False, float_format="%.10g")

g = ok.growth_12m
def gm(s): return None if pd.isna(lv.growth_12m.get(pd.Timestamp(s))) else round(float(lv.growth_12m[pd.Timestamp(s)]), 6)
last_level = lv.loc[:W1].cpi.dropna().index.max()
res = {
    "task_id": "F1-conf-01",
    "first_growth_month": ok.index.min().strftime("%Y-%m"),
    "last_month": last_level.strftime("%Y-%m"),
    "n_months_window": n_window,
    "n_months_growth": int(len(ok)),
    "n_months_missing_growth": int(n_window - len(ok)),
    "missing_growth_months": missing_g,
    "n_analysed": int(len(ok)),
    "mean_growth_12m": round(float(g.mean()), 6),
    "mean_growth_12m_2001_2013": round(float(g.loc["2001-01-01":"2013-12-01"].mean()), 6),
    "mean_growth_12m_2014_2026": round(float(g.loc["2014-01-01":"2026-09-01"].mean()), 6),
    "n_growth_2001_2013": int(g.loc["2001-01-01":"2013-12-01"].size),
    "n_growth_2014_2026": int(g.loc["2014-01-01":"2026-09-01"].size),
    "growth_2014_06": gm("2014-06-01"),
    "growth_2016_07": gm("2016-07-01"),
    "growth_2020_05": gm("2020-05-01"),
    "growth_2022_06": gm("2022-06-01"),
    "growth_2026_09": gm("2026-09-01"),
    "max_growth_12m": round(float(g.max()), 6),
    "max_growth_month": g.idxmax().strftime("%Y-%m"),
    "min_growth_12m": round(float(g.min()), 6),
    "min_growth_month": g.idxmin().strftime("%Y-%m"),
    "sd_growth_12m": round(float(g.std(ddof=1)), 6),
    "feasibility": "complete" if not missing_g else "partial",
    "feasibility_reason": None if not missing_g else
        f"g_t not computable for {', '.join(missing_g)}: the 2026-10-28 vintage reports no CPI level for 2025-10 and imputation is not allowed; all summaries use the {len(ok)} computable months.",
    "data_issues": issues,
    "n_input_rows": n_input_rows,
    "random_seed": None,
}
with open("out/results.json", "w") as f:
    json.dump(res, f, indent=2)
print(f"n_analysed={res['n_analysed']} missing={missing_g} mean={res['mean_growth_12m']:.4f}")