"""F1-conf-01: 12-month CPI inflation from two vintages. Run from workspace root."""
import json, os
import numpy as np, pandas as pd

D = "data"; O = "out"
F_OLD, F_NEW, F_AS = "cpi_vintage_2016-07-15.csv", "cpi_vintage_2026-10-28.csv", "cpi_assembled.csv"
W0, W1 = pd.Timestamp("2001-01-01"), pd.Timestamp("2026-09-01")

def load(f):
    d = pd.read_csv(os.path.join(D, f), dtype={"date": str})
    d["date"] = pd.to_datetime(d["date"]); return d

old, new = load(F_OLD), load(F_NEW)
asm = load(F_AS); asm["vintage"] = asm["vintage"].astype(str)
n_rows = {F_OLD: len(old), F_NEW: len(new), F_AS: len(asm)}
issues = []

# --- detected issues ---
miss_new = new[new.cpi.isna()].date.dt.strftime("%Y-%m").tolist()
if miss_new:
    issues.append({"issue": f"Vintage 2026-10-28 has no CPI value (empty cell) for {', '.join(miss_new)}.",
                   "handling": "Not imputed or back-filled from the older vintage (which does not cover it anyway); g_t for that month reported missing and counted; g_t twelve months later lies outside the window."})
ov = old.merge(new, on="date", suffixes=("_old", "_new"))
if len(ov):
    issues.append({"issue": f"The two vintages overlap for {len(ov)} months ({ov.date.min():%Y-%m}..{ov.date.max():%Y-%m}) with revised values (max abs difference {np.nanmax(np.abs(ov.cpi_old-ov.cpi_new)):.3f} index points).",
                   "handling": "Values from the most recent vintage (2026-10-28) used for overlapping months, per the information rule."})
    issues.append({"issue": "Growth rates for 2014-01..2014-12 divide a 2026-10-28-vintage level by a 2016-07-15-vintage level (different revision/seasonal-adjustment states).",
                   "handling": "Kept as required by the 'most recent vintage per month' rule; documented as a limitation."})
# assembled-file checks
truth = pd.concat([old[old.date < new.date.min()].assign(vintage="2016-07-15"), new.assign(vintage="2026-10-28")])
m = asm.merge(truth, on="date", how="outer", suffixes=("_as", "_tr"), indicator=True)
ovl = m[(m.date >= ov.date.min()) & (m.date <= ov.date.max())]
if (ovl.vintage_as == "2016-07-15").any():
    issues.append({"issue": f"cpi_assembled.csv uses the older 2016-07-15 vintage for {int((ovl.vintage_as=='2016-07-15').sum())} overlap months instead of the latest vintage.",
                   "handling": "Assembled file not used for estimation; levels taken directly from the vintage files."})
gaps = sorted(set(pd.date_range(asm.date.min(), asm.date.max(), freq="MS")) - set(asm.date))
tr_s = truth.set_index("date").cpi; as_s = asm.set_index("date").cpi
shifted = [d for d in tr_s.index if d >= pd.Timestamp("2016-07-01") and (d + pd.DateOffset(months=1)) in as_s.index
           and ((np.isclose(as_s[d + pd.DateOffset(months=1)], tr_s[d])) or (pd.isna(tr_s[d]) and pd.isna(as_s[d + pd.DateOffset(months=1)])))]
n_after = int((tr_s.index >= pd.Timestamp("2016-07-01")).sum())
if gaps or shifted:
    issues.append({"issue": f"cpi_assembled.csv omits month(s) {', '.join(g.strftime('%Y-%m') for g in gaps)} and {len(shifted)} of the {n_after} 2026-vintage values from 2016-07 onward appear one month late (series ends at a spurious 2026-10; the missing 2025-10 value appears as 2025-11).",
                   "handling": "Assembled file not used for estimation; levels taken directly from the vintage files."})

# --- merged series: latest vintage wins ---
s_old = old.set_index("date").cpi; s_new = new.set_index("date").cpi
idx = s_old.index.union(s_new.index)
P = pd.Series(np.nan, index=idx); V = pd.Series("", index=idx, dtype=object)
P[s_old.index] = s_old; V[s_old.index] = "2016-07-15"
P[s_new.index] = s_new; V[s_new.index] = "2026-10-28"   # covers month even if value empty -> no fallback
n_merged = len(P)
win = pd.date_range(W0, W1, freq="MS")
lag = P.reindex(win - pd.DateOffset(months=12)).values
a = pd.DataFrame({"date": win, "cpi": P.reindex(win).values, "cpi_lag12": lag, "vintage_used": V.reindex(win).values})
a["growth_12m"] = 100 * (a.cpi / a.cpi_lag12 - 1)

excl = []
def step(name, rule, before, after, why):
    excl.append(dict(step=name, rule=rule, n_before=before, n_after=after, n_removed=before - after, rationale=why))
step(1, "merge vintages (latest vintage per month) and restrict to window 2001-01..2026-09", n_merged, len(a),
     "months 1947-01..2000-12 outside the summary window (still used as 12-month lags)")
b1 = a[a.cpi.notna()]
step(2, "drop months with no index level P_t in the latest covering vintage", len(a), len(b1), "no interpolation/imputation allowed")
b2 = b1[b1.cpi_lag12.notna()]
step(3, "drop months with no index level P_{t-12}", len(b1), len(b2), "no interpolation/imputation allowed")
an = b2.copy()

g = an.set_index("date").growth_12m
def gm(d):
    t = pd.Timestamp(d + "-01"); return round(float(g[t]), 10) if t in g.index else None
sub1 = g[(g.index >= "2001-01-01") & (g.index <= "2013-12-01")]
sub2 = g[(g.index >= "2014-01-01") & (g.index <= "2026-09-01")]
n_missing = len(a) - len(an)
lastm = P[P.notna()].index.max()
res = {
    "task_id": "F1-conf-01",
    "first_growth_month": g.index.min().strftime("%Y-%m"),
    "last_month": lastm.strftime("%Y-%m"),
    "n_months_growth": int(len(an)),
    "n_months_missing_growth": int(n_missing),
    "n_analysed": int(len(an)),
    "mean_growth_12m": round(float(g.mean()), 10),
    "mean_growth_12m_2001_2013": round(float(sub1.mean()), 10),
    "mean_growth_12m_2014_2026": round(float(sub2.mean()), 10),
    "growth_2014_06": gm("2014-06"), "growth_2016_07": gm("2016-07"), "growth_2020_05": gm("2020-05"),
    "growth_2022_06": gm("2022-06"), "growth_2026_09": gm("2026-09"),
    "max_growth_12m": round(float(g.max()), 10), "max_growth_month": g.idxmax().strftime("%Y-%m"),
    "min_growth_12m": round(float(g.min()), 10), "min_growth_month": g.idxmin().strftime("%Y-%m"),
    "feasibility": "complete" if n_missing == 0 else "partial",
    "feasibility_reason": None if n_missing == 0 else
        f"g_t not computable for {n_missing} window month(s) ({', '.join(a[a.growth_12m.isna()].date.dt.strftime('%Y-%m'))}) because the source publishes no index level; means/extremes are over the {len(an)} computable months (2014-2026 sub-period: {len(sub2)} of {len(a[a.date>='2014-01-01'])} months).",
    "data_issues": issues,
    "n_input_rows": n_rows,
    "random_seed": None,
    "n_months_window": int(len(a)),
    "n_months_growth_2001_2013": int(len(sub1)),
    "n_months_growth_2014_2026": int(len(sub2)),
}
os.makedirs(O, exist_ok=True)
with open(os.path.join(O, "results.json"), "w") as f: json.dump(res, f, indent=2)
out = an.copy(); out["date"] = out.date.dt.strftime("%Y-%m-01")
out.to_csv(os.path.join(O, "analytic.csv"), index=False, float_format="%.10g")
pd.DataFrame(excl).to_csv(os.path.join(O, "exclusions.csv"), index=False)
print(json.dumps({k: v for k, v in res.items() if k != "data_issues"}, indent=1))