"""E8 evaluation: apply the frozen dev-fitted policy to held-out records (full factorial on test instances -> the policy's outcome on an
instance is the recorded outcome of the condition it picks). Includes fallback counting and router overhead (feature lookup time)."""
from __future__ import annotations
import json, time, yaml
import numpy as np, pandas as pd
from .util import ROOT, read_json, write_json
from .analysis.load import load_experiments
from .analysis.stats import cluster_bootstrap_mean, paired_difference
from .router import choose

def main(config_path: str) -> int:
    cfg = yaml.safe_load(open(config_path))
    pol = read_json(ROOT / "runs/checkpoints/router_policy.json")
    df = load_experiments(cfg["test_patterns"]); df = df[df["split"] == "test"]
    df["question_access_group"] = df["question_access"].replace({"none": "blind"})
    df = df[df["condition"].isin(pol["actions"])]
    rows = []
    t_router = []
    for (doc, q, pair), g in df.groupby(["document_id", "question_id", "pair"]):
        ctx = g["message_utf8_bytes"].max() / 4.0 if "A1" in set(g["method_id"]) else g["serialized_payload_bytes"].max() / 4.0
        fam = g["task_family"].iloc[0]
        for acc in ["blind", "aware"]:
            gg = g[g["question_access_group"] == acc] if acc == "aware" else g[g["question_access_group"].isin(["blind"])]
            if gg.empty: continue
            t0 = time.perf_counter(); action, fb = choose(pol, fam, ctx, pair, acc); t_router.append((time.perf_counter() - t0) * 1000)
            hit = gg[gg["condition"] == action]
            if hit.empty:   # chosen action not run on this instance -> fallback to best static that exists here
                action = pol["global_best_static"]; fb = True; hit = gg[gg["condition"] == action]
                if hit.empty: continue
            r = hit.iloc[0]
            rows.append({"document_id": doc, "question_id": q, "pair": pair, "task_family": fam, "access": acc, "policy": "router", "condition": action, "fallback": fb,
                         "quality_operational": r["quality_operational"], "cost": r[pol["cost_col"]], "bytes": r["serialized_payload_bytes"], "router_ms": t_router[-1]})
            # static baselines on the same instance
            best_acc = pol.get("global_best_static_by_access", {}).get(acc, pol["global_best_static"])
            for name, cond in [("best_static_dev", best_acc), ("full_context", "A1"), ("structured", "A5|budget_tokens=512|blind"), ("native_cache", "B0"), ("attention_cache_aware", "B5|keep_frac=0.25|aware")]:
                h = gg[gg["condition"] == cond]
                if not h.empty:
                    rows.append({"document_id": doc, "question_id": q, "pair": pair, "task_family": fam, "access": acc, "policy": name, "condition": cond, "fallback": False,
                                 "quality_operational": h.iloc[0]["quality_operational"], "cost": h.iloc[0][pol["cost_col"]], "bytes": h.iloc[0]["serialized_payload_bytes"], "router_ms": 0.0})
            # oracle (infeasible upper reference)
            best = gg.loc[gg["quality_operational"].idxmax()]
            rows.append({"document_id": doc, "question_id": q, "pair": pair, "task_family": fam, "access": acc, "policy": "oracle_infeasible", "condition": best["condition"], "fallback": False,
                         "quality_operational": best["quality_operational"], "cost": best[pol["cost_col"]], "bytes": best["serialized_payload_bytes"], "router_ms": 0.0})
    out = pd.DataFrame(rows)
    summ = []
    for (pol_name, acc), g in out.groupby(["policy", "access"]):
        q = cluster_bootstrap_mean(g, "quality_operational"); c = cluster_bootstrap_mean(g, "cost")
        summ.append({"policy": pol_name, "access": acc, "quality": q["mean"], "q_lo": q["lo"], "q_hi": q["hi"], "cost_ms": c["mean"], "c_lo": c["lo"], "c_hi": c["hi"], "bytes_median": g["bytes"].median(),
                     "fallback_rate": g["fallback"].mean(), "router_ms_mean": g["router_ms"].mean(), "n": q["n"], "n_docs": q["n_clusters"]})
    S = pd.DataFrame(summ)
    tests = []
    for acc in ["blind", "aware"]:
        sub = out[out["access"] == acc]
        tests.append({**paired_difference(sub, "quality_operational", "policy", "router", "best_static_dev", keys=("document_id", "question_id", "pair")), "access": acc, "metric": "quality"})
        tests.append({**paired_difference(sub, "cost", "policy", "router", "best_static_dev", keys=("document_id", "question_id", "pair")), "access": acc, "metric": "cost_ms"})
    (ROOT / "results/tables").mkdir(parents=True, exist_ok=True)
    S.to_csv(ROOT / "results/tables/E8_router_summary.csv", index=False); pd.DataFrame(tests).to_csv(ROOT / "results/tables/E8_router_tests.csv", index=False)
    out.to_csv(ROOT / "results/tables/E8_router_instances.csv", index=False)
    print(S.to_string()); print(pd.DataFrame(tests)[["access", "metric", "diff", "lo", "hi", "p", "n"]].to_string())
    return 0