1"""E8 evaluation: apply the frozen dev-fitted policy to held-out records (full factorial on test instances -> the policy's outcome on an2instance is the recorded outcome of the condition it picks). Includes fallback counting and router overhead (feature lookup time)."""3from __future__ import annotations4import json, time, yaml5import numpy as np, pandas as pd6from .util import ROOT, read_json, write_json7from .analysis.load import load_experiments8from .analysis.stats import cluster_bootstrap_mean, paired_difference9from .router import choose1011def main(config_path: str) -> int:12 cfg = yaml.safe_load(open(config_path))13 pol = read_json(ROOT / "runs/checkpoints/router_policy.json")14 df = load_experiments(cfg["test_patterns"]); df = df[df["split"] == "test"]15 df["question_access_group"] = df["question_access"].replace({"none": "blind"})16 df = df[df["condition"].isin(pol["actions"])]17 rows = []18 t_router = []19 for (doc, q, pair), g in df.groupby(["document_id", "question_id", "pair"]):20 ctx = g["message_utf8_bytes"].max() / 4.0 if "A1" in set(g["method_id"]) else g["serialized_payload_bytes"].max() / 4.021 fam = g["task_family"].iloc[0]22 for acc in ["blind", "aware"]:23 gg = g[g["question_access_group"] == acc] if acc == "aware" else g[g["question_access_group"].isin(["blind"])]24 if gg.empty: continue25 t0 = time.perf_counter(); action, fb = choose(pol, fam, ctx, pair, acc); t_router.append((time.perf_counter() - t0) * 1000)26 hit = gg[gg["condition"] == action]27 if hit.empty: # chosen action not run on this instance -> fallback to best static that exists here28 action = pol["global_best_static"]; fb = True; hit = gg[gg["condition"] == action]29 if hit.empty: continue30 r = hit.iloc[0]31 rows.append({"document_id": doc, "question_id": q, "pair": pair, "task_family": fam, "access": acc, "policy": "router", "condition": action, "fallback": fb,32 "quality_operational": r["quality_operational"], "cost": r[pol["cost_col"]], "bytes": r["serialized_payload_bytes"], "router_ms": t_router[-1]})33 # static baselines on the same instance34 best_acc = pol.get("global_best_static_by_access", {}).get(acc, pol["global_best_static"])35 for name, cond in [("best_static_dev", best_acc), ("full_context", "A1"), ("structured", "A5|budget_tokens=512|blind"), ("native_cache", "B0"), ("attention_cache_aware", "B5|keep_frac=0.25|aware")]:36 h = gg[gg["condition"] == cond]37 if not h.empty:38 rows.append({"document_id": doc, "question_id": q, "pair": pair, "task_family": fam, "access": acc, "policy": name, "condition": cond, "fallback": False,39 "quality_operational": h.iloc[0]["quality_operational"], "cost": h.iloc[0][pol["cost_col"]], "bytes": h.iloc[0]["serialized_payload_bytes"], "router_ms": 0.0})40 # oracle (infeasible upper reference)41 best = gg.loc[gg["quality_operational"].idxmax()]42 rows.append({"document_id": doc, "question_id": q, "pair": pair, "task_family": fam, "access": acc, "policy": "oracle_infeasible", "condition": best["condition"], "fallback": False,43 "quality_operational": best["quality_operational"], "cost": best[pol["cost_col"]], "bytes": best["serialized_payload_bytes"], "router_ms": 0.0})44 out = pd.DataFrame(rows)45 summ = []46 for (pol_name, acc), g in out.groupby(["policy", "access"]):47 q = cluster_bootstrap_mean(g, "quality_operational"); c = cluster_bootstrap_mean(g, "cost")48 summ.append({"policy": pol_name, "access": acc, "quality": q["mean"], "q_lo": q["lo"], "q_hi": q["hi"], "cost_ms": c["mean"], "c_lo": c["lo"], "c_hi": c["hi"], "bytes_median": g["bytes"].median(),49 "fallback_rate": g["fallback"].mean(), "router_ms_mean": g["router_ms"].mean(), "n": q["n"], "n_docs": q["n_clusters"]})50 S = pd.DataFrame(summ)51 tests = []52 for acc in ["blind", "aware"]:53 sub = out[out["access"] == acc]54 tests.append({**paired_difference(sub, "quality_operational", "policy", "router", "best_static_dev", keys=("document_id", "question_id", "pair")), "access": acc, "metric": "quality"})55 tests.append({**paired_difference(sub, "cost", "policy", "router", "best_static_dev", keys=("document_id", "question_id", "pair")), "access": acc, "metric": "cost_ms"})56 (ROOT / "results/tables").mkdir(parents=True, exist_ok=True)57 S.to_csv(ROOT / "results/tables/E8_router_summary.csv", index=False); pd.DataFrame(tests).to_csv(ROOT / "results/tables/E8_router_tests.csv", index=False)58 out.to_csv(ROOT / "results/tables/E8_router_instances.csv", index=False)59 print(S.to_string()); print(pd.DataFrame(tests)[["access", "metric", "diff", "lo", "hi", "p", "n"]].to_string())60 return 061