1"""Build every empirical table and figure from raw records. Each output gets a provenance entry (source run files + script)."""2from __future__ import annotations3import json, sys, glob4from pathlib import Path5import numpy as np, pandas as pd6import matplotlib7matplotlib.use("Agg")8import matplotlib.pyplot as plt9from ..util import ROOT, read_json, write_json, read_jsonl, utc_now10from .load import load_experiments, LABELS11from .stats import cluster_bootstrap_mean, paired_difference, holm, summarize_latency1213# fixed hue per method family (colorblind-validated palette; see results/figures/PALETTE.txt)14PAL = {"A": "#2a78d6", "B_quant": "#eb6834", "B_sel": "#1baf7a", "B_ref": "#6250d6", "C": "#e87ba4", "A_struct": "#eda100", "neg": "#e34948", "hosted": "#008300"}15METHOD_COLOR = {"A0": "#7a7a78", "A1": "#2a78d6", "A2": "#2a78d6", "A3": "#2a78d6", "A4": "#2a78d6", "A5": "#eda100", "A6": "#eda100", "A7": "#2a78d6", "A8": "#2a78d6",16 "B0": "#6250d6", "B1": "#6250d6", "B2": "#eb6834", "B3": "#1baf7a", "B4": "#e34948", "B5": "#1baf7a", "B6": "#1baf7a", "B9": "#eb6834", "C0": "#e34948", "C2": "#e87ba4", "C3": "#e87ba4"}17MARKER = {"A0": "x", "A1": "o", "A2": "v", "A3": "^", "A4": "s", "A5": "D", "A6": "P", "A7": "o", "A8": "<", "B0": "o", "B1": "o", "B2": "s", "B3": "^", "B4": "x", "B5": "D", "B6": "P", "B9": "*", "C0": "x", "C2": "s", "C3": "D"}18TAB = ROOT / "results/tables"; FIG = ROOT / "results/figures"19PROV: dict = {}2021def _prov(path: Path, sources: list[str], note: str = ""):22 PROV[str(path.relative_to(ROOT))] = {"sources": sources, "script": "src/wp30/analysis/analyze.py", "generated_utc": utc_now(), "note": note}2324def _save_fig(fig, name: str, sources: list[str], note: str = ""):25 FIG.mkdir(parents=True, exist_ok=True)26 for ext in ("pdf", "png"):27 p = FIG / f"{name}.{ext}"; fig.savefig(p, bbox_inches="tight", dpi=160 if ext == "png" else None)28 _prov(p, sources, note)29 plt.close(fig)3031def _style():32 plt.rcParams.update({"font.size": 9, "axes.spines.top": False, "axes.spines.right": False, "axes.grid": True, "grid.alpha": 0.25, "grid.linewidth": 0.5,33 "legend.frameon": False, "figure.dpi": 110, "axes.titlesize": 9.5, "axes.labelsize": 9})3435def fmt_bytes(b):36 return f"{b/1e6:.1f} MB" if b >= 1e6 else (f"{b/1e3:.1f} kB" if b >= 1e3 else f"{b:.0f} B")3738# ------------------------------------------------------------------------------------------------------------- tables39def table_primary(df: pd.DataFrame, exp: str, sources: list[str]) -> pd.DataFrame:40 rows = []41 for (fam, pair, cond), g in df.groupby(["task_family", "pair", "condition"]):42 q = cluster_bootstrap_mean(g, "quality_operational"); ok = (g["status"] == "success").mean()43 rows.append({"family": fam, "pair": pair, "condition": cond, "method": g["method_id"].iloc[0], "access": g["question_access"].iloc[0], "n": q["n"], "n_docs": q["n_clusters"],44 "quality": q["mean"], "q_lo": q["lo"], "q_hi": q["hi"], "success_rate": ok, "payload_bytes_median": g["serialized_payload_bytes"].median(),45 "cache_bytes_median": g["cache_tensor_bytes"].median(), "cold_ms_p50": g["cost_cold_ms"].median(), "warm_ms_p50": g["cost_warm_query_ms"].median(),46 "amortized_ms_p50": g["cost_amortized_ms"].median(), "ttft_ms_p50": g["ttft_ms"].median(), "peak_decode_mem_median": g["peak_decode_memory"].median(),47 "fetched_bytes_mean": g["fetched_bytes"].mean()})48 t = pd.DataFrame(rows).sort_values(["family", "pair", "condition"])49 TAB.mkdir(parents=True, exist_ok=True); p = TAB / f"{exp}_primary.csv"; t.to_csv(p, index=False); _prov(p, sources, "per-condition quality (cluster bootstrap CI) and resource medians")50 return t5152def table_failures(df: pd.DataFrame, exp: str, sources):53 t = df.groupby(["method_id", "status"]).size().unstack(fill_value=0)54 p = TAB / f"{exp}_failures.csv"; t.to_csv(p); _prov(p, sources, "attempt status counts by method"); return t5556def table_qtype(df: pd.DataFrame, exp: str, sources):57 d = df[df["task_family"] == "synthetic"]58 rows = []59 for (cond, qt), g in d.groupby(["condition", "qtype"]):60 q = cluster_bootstrap_mean(g, "quality_operational", n_boot=500); rows.append({"condition": cond, "qtype": qt, "quality": q["mean"], "lo": q["lo"], "hi": q["hi"], "n": q["n"]})61 t = pd.DataFrame(rows); p = TAB / f"{exp}_by_question_type.csv"; t.to_csv(p, index=False); _prov(p, sources, "synthetic accuracy by question type"); return t6263def primary_tests(df: pd.DataFrame, exp: str, sources, tests: list[dict]):64 rows = []65 for t in tests:66 sub = df[df["task_family"].isin(t["families"])] if t.get("families") else df67 if t.get("pair"): sub = sub[sub["pair"] == t["pair"]]68 r = paired_difference(sub, t.get("value", "quality_operational"), "condition", t["a"], t["b"], keys=("document_id", "question_id", "pair"))69 r.update({"name": t["name"], "hypothesis": t.get("hypothesis", "")}); rows.append(r)70 out = pd.DataFrame(rows)71 if len(out):72 out["p_holm"] = holm([p if not np.isnan(p) else 1.0 for p in out["p"]])73 p = TAB / f"{exp}_primary_tests.csv"; out.to_csv(p, index=False); _prov(p, sources, "registered primary comparisons, paired, cluster bootstrap, Holm"); return out7475# ------------------------------------------------------------------------------------------------------------ figures76def _short_pair(p: str) -> str:77 return p.replace("Qwen2.5-3B-Instruct-bf16->Qwen2.5-3B-Instruct-bf16", "Qwen2.5-3B").replace("Llama-3.2-3B-Instruct-bf16->Llama-3.2-3B-Instruct-bf16", "Llama-3.2-3B").replace("Qwen2.5-1.5B-Instruct-bf16->Qwen2.5-3B-Instruct-bf16", "Qwen1.5B->3B")7879def _mk(m: str, bits) -> str:80 if m == "B2" and bits == 4: return "p"81 if m == "A7": return "h"82 return MARKER.get(m, "o")8384def fig_quality_vs_bytes(df: pd.DataFrame, exp: str, sources, access_filter=None, name="fig2_quality_vs_bytes"):85 _style()86 fams = [f for f in ["synthetic", "exec", "public_qasper", "public_2wikimqa"] if f in set(df["task_family"])]87 pairs = sorted(df["pair"].unique())88 fig, axes = plt.subplots(len(pairs), len(fams), figsize=(3.1 * len(fams), 2.7 * len(pairs)), squeeze=False, sharey="row")89 for i, pair in enumerate(pairs):90 for j, fam in enumerate(fams):91 ax = axes[i][j]; d = df[(df["pair"] == pair) & (df["task_family"] == fam)]92 if access_filter: d = d[d["question_access"].isin(access_filter)]93 for cond, g in d.groupby("condition"):94 m = g["method_id"].iloc[0]; q = cluster_bootstrap_mean(g, "quality_operational", n_boot=300)95 x = g["serialized_payload_bytes"].median()96 if m == "A0" or x is None or np.isnan(x): continue97 ax.errorbar(max(x, 1), q["mean"], yerr=[[q["mean"] - q["lo"]], [q["hi"] - q["mean"]]], fmt=_mk(m, g["bits"].iloc[0]), color=METHOD_COLOR.get(m, "#444"), ms=4.5, lw=0.9, capsize=1.5, mfc="white" if m.startswith("B") else None)98 ax.set_xscale("log"); ax.set_title(f"{fam.replace('public_', '')} - {_short_pair(pair)}", fontsize=8.5); ax.set_ylim(-0.03, 1.03)99 if i == len(pairs) - 1: ax.set_xlabel("serialized payload bytes (log)")100 if j == 0: ax.set_ylabel("quality (acc. or F1)")101 present = set(df["method_id"]); has4 = ((df["method_id"] == "B2") & (df["bits"] == 4)).any()102 handles = [plt.Line2D([], [], marker=_mk(m, None), color=METHOD_COLOR[m], ls="", ms=5, mfc="white" if m.startswith("B") else None, label=LABELS[m] + (" 8-bit" if m == "B2" else "")) for m in ["A1", "A2", "A3", "A4", "A5", "A6", "A7", "A8", "B0", "B1", "B2", "B3", "B4", "B5", "B6", "B9", "C0", "C2", "C3"] if m in present]103 if has4: handles.append(plt.Line2D([], [], marker="p", color=METHOD_COLOR["B2"], ls="", ms=5, mfc="white", label="B2 packed quant 4-bit"))104 fig.legend(handles=handles, loc="lower center", ncol=5, fontsize=7, bbox_to_anchor=(0.5, -0.06 - 0.02 * (len(handles) // 5)))105 fig.subplots_adjust(hspace=0.5); fig.suptitle("Quality vs measured serialized payload bytes (medians; 95% cluster-bootstrap CIs)", fontsize=9.5, y=1.0)106 _save_fig(fig, name, sources, "points = conditions; x = median payload bytes; y = mean quality")107108def fig_quality_vs_latency(df: pd.DataFrame, exp: str, sources, name="fig3_quality_vs_latency_memory"):109 _style()110 fams = [f for f in ["synthetic", "exec", "public_qasper", "public_2wikimqa"] if f in set(df["task_family"])]111 fig, axes = plt.subplots(2, len(fams), figsize=(3.1 * len(fams), 5.2), squeeze=False)112 for j, fam in enumerate(fams):113 d = df[df["task_family"] == fam]114 for cond, g in d.groupby("condition"):115 m = g["method_id"].iloc[0]; q = g["quality_operational"].mean()116 if m == "A0": continue117 axes[0][j].scatter(g["cost_cold_ms"].median(), q, marker=_mk(m, g["bits"].iloc[0]), color=METHOD_COLOR.get(m, "#444"), s=22, facecolors="white" if m.startswith("B") else METHOD_COLOR.get(m, "#444"), lw=0.9)118 axes[1][j].scatter(g["peak_decode_memory"].median() / 1e9, q, marker=_mk(m, g["bits"].iloc[0]), color=METHOD_COLOR.get(m, "#444"), s=22, facecolors="white" if m.startswith("B") else METHOD_COLOR.get(m, "#444"), lw=0.9)119 axes[0][j].set_xscale("log"); axes[0][j].set_title(fam.replace("public_", ""), fontsize=8.5); axes[0][j].set_xlabel("cold end-to-end latency, ms (log)"); axes[1][j].set_xlabel("peak MLX memory during trial, GB")120 axes[0][j].set_ylim(-0.03, 1.03); axes[1][j].set_ylim(-0.03, 1.03)121 axes[0][0].set_ylabel("quality"); axes[1][0].set_ylabel("quality")122 fig.suptitle("Quality vs cold latency (top) and peak memory (bottom); all model pairs pooled", fontsize=9.5)123 _save_fig(fig, name, sources)124125def fig_latency_decomposition(df: pd.DataFrame, exp: str, sources, name="fig4_latency_decomposition"):126 _style()127 d = df[(df["task_family"] == "synthetic") & (df["status"] == "success")]128 conds = [c for c in ["A1", "A4|budget_tokens=512|blind", "A5|budget_tokens=512|blind", "A7|codec=zstd", "B0", "B1", "B2|bits=8", "B3|keep_frac=0.25", "B5|keep_frac=0.25|aware"] if c in set(d["condition"])]129 comps = [("span_encode_ms", "sender encode (text)"), ("span_doc_prefill_ms", "document prefill (cache)"), ("span_compress_ms", "compress/select"), ("span_serialize_ms", "serialize"), ("span_load_ms", "deserialize/load"), ("span_receiver_prefill_ms", "receiver prefill"), ("ttft_ms", "query prefill→first token"), ("span_decode_ms", "decode")]130 cols = ["#2a78d6", "#6250d6", "#1baf7a", "#eda100", "#e87ba4", "#eb6834", "#008300", "#7a7a78"]131 fig, ax = plt.subplots(figsize=(7.2, 3.2))132 for i, c in enumerate(conds):133 g = d[d["condition"] == c]; left = 0134 for (col, lab), color in zip(comps, cols):135 v = g[col].median() if col in g and g[col].notna().any() else 0.0136 if v and v > 0:137 ax.barh(i, v, left=left, color=color, height=0.6, label=lab if i == 0 else None, edgecolor="white", linewidth=0.8); left += v138 ax.set_yticks(range(len(conds))); ax.set_yticklabels([c.replace("|", " ") for c in conds], fontsize=7.5); ax.invert_yaxis(); ax.set_xlabel("median span duration, ms (serial accounting; one question per document build)")139 ax.legend(fontsize=7, ncol=4, loc="lower right"); ax.set_title("Latency decomposition — synthetic, cold scenario (document build counted once)", fontsize=9)140 _save_fig(fig, name, sources)141142def fig_handoffs(df: pd.DataFrame, exp: str, sources, name="fig5_handoffs"):143 _style()144 d = df[df["topology"].astype(str).str.startswith("chain")]145 if d.empty: return146 fig, axes = plt.subplots(1, 3, figsize=(9.6, 2.8))147 for cid, g in d.groupby("run_group_id"):148 pass149 d["chain"] = d["run_group_id"].str.split(":").str[1]150 for chain, g in d.groupby("chain"):151 rep = g["method_id"].iloc[0]; mode = g["topology"].iloc[0].replace("chain_", "")152 color = "#eda100" if rep == "A5" else "#2a78d6"; ls = "-" if mode == "closed" else "--"153 hop = g.groupby("handoff_index")154 ret = hop["retention_rate"].mean(); uns = hop["unsupported"].mean(); acc = hop["quality_operational"].mean()155 axes[0].plot(ret.index, ret.values, ls, color=color, marker="o", ms=4, label=f"{rep} {mode}")156 axes[1].plot(uns.index, uns.values, ls, color=color, marker="o", ms=4)157 axes[2].plot(acc.index, acc.values, ls, color=color, marker="o", ms=4)158 for ax, t in zip(axes, ["retained correct facts (ledger)", "unsupported numbers + identifiers per message", "question accuracy from message"]):159 ax.set_xlabel("handoffs"); ax.set_title(t, fontsize=8.5); ax.set_xticks(sorted(d["handoff_index"].unique()))160 axes[0].set_ylim(0, 1.02); axes[2].set_ylim(0, 1.02); axes[0].legend(fontsize=7)161 _save_fig(fig, name, sources, "means over documents; chains defined in configs/chains*.yaml")162163def fig_breakeven(net_rows: list[dict], exp: str, sources, name="fig6_breakeven"):164 """Left: measured two-node transfer vs remote text prefill (medians with p25-p75 bars per target length).165 Right: analytic transfer-time model from the measured bandwidth and the measured remote prefill slope."""166 _style()167 if not net_rows: return168 n = pd.DataFrame(net_rows); n = n[n["label"] != "loopback"]169 fig, axes = plt.subplots(1, 2, figsize=(8.6, 3.1))170 ax = axes[0]171 series = [("text_remote_prefill", None, "none", "text -> remote prefill", "#2a78d6", "o", "-"),172 ("B1", None, "none", "B1 bf16 cache", "#6250d6", "D", "-"), ("B1", None, "zstd", "B1 bf16 + zstd", "#6250d6", "D", ":"),173 ("B2", 8.0, "none", "B2 8-bit cache", "#eb6834", "s", "-"), ("B2", 4.0, "none", "B2 4-bit cache (Qwen: quality 0)", "#e34948", "x", "--")]174 for meth, bits, codec, lab, color, mk, ls in series:175 g = n[(n["method"] == meth) & (n["codec"].fillna("none") == codec)]176 g = g[g["bits"].isna()] if bits is None else g[g["bits"] == bits]177 if g.empty: continue178 agg = g.groupby("target_tokens").agg(x=("prefix_tokens", "median"), y=("roundtrip_ms", "median"), lo=("roundtrip_ms", lambda s: s.quantile(0.25)), hi=("roundtrip_ms", lambda s: s.quantile(0.75))).reset_index()179 ax.errorbar(agg["x"], agg["y"], yerr=[agg["y"] - agg["lo"], agg["hi"] - agg["y"]], fmt=mk + ls, color=color, ms=4.5, lw=1.1, capsize=2, label=lab, mfc="white" if meth == "B1" else None)180 loc = n[n["method"] == "local_prefill"]181 if len(loc):182 agg = loc.groupby("target_tokens").agg(x=("prefix_tokens", "median"), y=("local_prefill_ms", "median")).reset_index()183 ax.plot(agg["x"], agg["y"], "^-.", color="#7a7a78", ms=4.5, lw=1, label="local prefill only (no transfer, no decode)")184 ax.set_xlabel("context tokens (median per length class)"); ax.set_ylabel("ms"); ax.set_title("Measured: M3U96a -> M3U96b, 1 GbE (18 runs per point)", fontsize=8.5)185 ax.legend(fontsize=6.5, loc="upper left")186 ax = axes[1]187 bw = n[n["method"].isin(["B1", "B2"])]["effective_gbps"].median()188 b1 = n[(n["method"] == "B1") & (n["codec"].fillna("none") == "none")]189 bpt = (b1["serialized_bytes"] / b1["prefix_tokens"]).median() if len(b1) else np.nan190 tr = n[n["method"] == "text_remote_prefill"]191 if len(tr) and not np.isnan(bpt):192 slope = (tr["remote_prefill_ms"] / tr["prefix_tokens"]).median()193 L = np.linspace(500, 32000, 50)194 for frac, lab, color in [(1.0, "bf16 cache", "#6250d6"), (0.53, "8-bit cache", "#eb6834")]:195 for bw_g, ls, alpha in [(bw, "-", 0.95), (10.0, "--", 0.5)]:196 ax.plot(L, L * bpt * frac * 8 / (bw_g * 1e9) * 1000, ls, color=color, lw=1.1, alpha=alpha, label=f"{lab} transfer @ {bw_g:.1f} Gb/s" + (" (hypothetical)" if bw_g == 10.0 else " (measured)"))197 ax.plot(L, L * slope, "-", color="#2a78d6", lw=1.6, label="remote text prefill (measured slope, M3 Ultra)")198 ax.set_xlabel("context tokens"); ax.set_ylabel("ms"); ax.set_title("Analytic model (Qwen2.5-3B cache, measured slopes)", fontsize=8.5)199 ax.legend(fontsize=6.3, loc="upper left"); ax.set_ylim(0, None)200 _save_fig(fig, name, sources, "left = measured (medians, p25-p75); right = analytic model from measured bandwidth and prefill slope; dashed 10 Gb/s is hypothetical")201202# ------------------------------------------------------------------------------------------------------------ main203def main(protocol_path: str) -> int:204 _style()205 prot = read_json(ROOT / protocol_path) if (ROOT / protocol_path).exists() else {}206 core_exp = prot.get("core_experiment_ids", ["E1_core"]) if prot else ["E1_core"]207 exps = {p.stem: p.name for p in (ROOT / "runs/raw").glob("*.jsonl")}208 (FIG).mkdir(parents=True, exist_ok=True); (TAB).mkdir(parents=True, exist_ok=True)209 (FIG / "PALETTE.txt").write_text("categorical palette validated with dataviz validate_palette.js (light): #2a78d6,#eb6834,#1baf7a,#eda100,#e87ba4,#008300,#6250d6,#e34948 — ALL CHECKS PASS\n")210 # choose the experiment set to analyze: core if present, else pilot211 main_exp = "E1_core" if "E1_core" in exps else ("E1_pilot" if "E1_pilot" in exps else None)212 if main_exp:213 src = [f"runs/raw/{main_exp}.jsonl"]214 df = load_experiments([f"{main_exp}.jsonl"]); df_all = load_experiments([f"{main_exp}.jsonl"], last_attempt_only=False)215 table_primary(df, main_exp, src); table_failures(df_all, main_exp, src); table_qtype(df, main_exp, src)216 tests = [217 {"name": "T1 A5 vs A4 (512, blind, synthetic)", "a": "A5|budget_tokens=512|blind", "b": "A4|budget_tokens=512|blind", "families": ["synthetic"], "hypothesis": "RQ5/H1"},218 {"name": "T2 B2-8bit vs B0", "a": "B2|bits=8", "b": "B0", "hypothesis": "H2"},219 {"name": "T3 B2-4bit vs B0", "a": "B2|bits=4", "b": "B0", "hypothesis": "H2"},220 {"name": "T4 B5-aware vs B3 (1/4)", "a": "B5|keep_frac=0.25|aware", "b": "B3|keep_frac=0.25", "hypothesis": "H3"},221 {"name": "T6 A1 vs B0 (text vs native cache, same content)", "a": "A1", "b": "B0", "hypothesis": "consistency"},222 ]223 primary_tests(df, main_exp, src, tests)224 fig_quality_vs_bytes(df, main_exp, src)225 fig_quality_vs_bytes(df[df["question_access"].isin(["blind", "none"])], main_exp, src, name="figA_quality_vs_bytes_blind")226 fig_quality_vs_bytes(df[df["question_access"] == "aware"], main_exp, src, name="figA_quality_vs_bytes_aware")227 fig_quality_vs_latency(df, main_exp, src); fig_latency_decomposition(df, main_exp, src)228 chain_exp = "E4_core" if "E4_core" in exps else ("E4_pilot" if "E4_pilot" in exps else None)229 if chain_exp:230 dfc = load_experiments([f"{chain_exp}.jsonl"]); src = [f"runs/raw/{chain_exp}.jsonl"]231 from .chains_recompute import recompute232 dfc = recompute(dfc, chain_exp) # ledger metrics recomputed from stored hop messages (span-offset fix)233 fig_handoffs(dfc, chain_exp, src)234 rows = []235 for (grp, h), g in dfc.groupby(["run_group_id", "handoff_index"]):236 q = cluster_bootstrap_mean(g, "quality_operational", n_boot=500); r = cluster_bootstrap_mean(g.drop_duplicates("document_id"), "retention_rate", n_boot=500)237 rows.append({"chain": grp.split(":")[1], "handoffs": h, "accuracy": q["mean"], "acc_lo": q["lo"], "acc_hi": q["hi"], "retention": r["mean"], "ret_lo": r["lo"], "ret_hi": r["hi"],238 "unsupported_mean": g.drop_duplicates("document_id")["unsupported"].mean(), "fetched_bytes_mean": g["fetched_bytes"].mean(), "msg_bytes_mean": g["msg_bytes_hop"].mean(), "n": q["n"], "n_docs": q["n_clusters"]})239 t = pd.DataFrame(rows); p = TAB / f"{chain_exp}_handoffs.csv"; t.to_csv(p, index=False); _prov(p, src, "repeated handoffs")240 primary_tests(dfc[dfc["handoff_index"] == 4].assign(condition=dfc["run_group_id"].str.split(":").str[1]), chain_exp + "_h4", src,241 [{"name": "T5 A5 chain vs A4 chain at 4 handoffs (closed)", "a": "A5|closed|qwen3b", "b": "A4|closed|qwen3b", "hypothesis": "H5"}])242 fan_exp = "E5_core" if "E5_core" in exps else ("E5_pilot" if "E5_pilot" in exps else None)243 if fan_exp:244 dff = load_experiments([f"{fan_exp}.jsonl"]); src = [f"runs/raw/{fan_exp}.jsonl"]245 rows = []246 for (cond, qt), g in dff.groupby(["condition", "qtype"]):247 q = cluster_bootstrap_mean(g, "quality_operational", n_boot=500); rows.append({"condition": cond, "qtype": qt, "quality": q["mean"], "lo": q["lo"], "hi": q["hi"], "n": q["n"], "payload_bytes_median": g["serialized_payload_bytes"].median(), "fetched_bytes_mean": g["fetched_bytes"].mean()})248 t = pd.DataFrame(rows); p = TAB / f"{fan_exp}_fanin.csv"; t.to_csv(p, index=False); _prov(p, src, "fan-in by method and question type")249 net_files = [p for p in (ROOT / "runs/raw").glob("E6_*.jsonl") if "smoke" not in p.name] or list((ROOT / "runs/raw").glob("E6_*.jsonl"))250 if net_files:251 rows = [r for p in net_files for r in read_jsonl(p)]252 fig_breakeven(rows, "E6", [str(p.relative_to(ROOT)) for p in net_files])253 n = pd.DataFrame(rows)254 agg = n.groupby(["label", "method", "bits", "codec", "target_tokens"], dropna=False).agg(n=("repeat", "size"), bytes=("serialized_bytes", "median"), roundtrip_ms_p50=("roundtrip_ms", "median"), send_ms_p50=("send_ms", "median"),255 remote_load_ms_p50=("remote_load_ms", "median"), remote_ttft_p50=("remote_ttft_ms", "median"), remote_prefill_p50=("remote_prefill_ms", "median"), eff_gbps=("effective_gbps", "median"), correct=("correct", "mean"),256 local_prefill_p50=("local_prefill_ms", "median"), local_ttft_p50=("local_ttft_ms", "median"), remote_decode_p50=("remote_decode_ms", "median")).reset_index()257 p = TAB / "E6_network.csv"; agg.to_csv(p, index=False); _prov(p, [str(x.relative_to(ROOT)) for x in net_files], "two-node transfer measurements")258 hosted = [p for p in (ROOT / "runs/raw").glob("ED_hosted_core.jsonl")] or [p for p in (ROOT / "runs/raw").glob("ED_hosted_pilot.jsonl")]259 if hosted:260 dh = load_experiments([p.name for p in hosted]); src = [str(p.relative_to(ROOT)) for p in hosted]261 rows = []262 for (fam, recv, cond), g in dh.groupby(["task_family", "receiver", "condition"]):263 q = cluster_bootstrap_mean(g, "quality_operational", n_boot=500)264 rows.append({"family": fam, "provider_model": recv, "condition": cond, "quality": q["mean"], "lo": q["lo"], "hi": q["hi"], "n": q["n"], "cost_usd_mean": g["cost_usd"].mean(), "ttft_ms_p50": g["ttft_ms"].median(), "total_ms_p50": g["total_wall_ms"].median(), "success_rate": (g["status"] == "success").mean()})265 t = pd.DataFrame(rows); p = TAB / "ED_hosted.csv"; t.to_csv(p, index=False); _prov(p, src, "hosted receivers; cost estimated from provider price tables")266 r8 = TAB / "E8_router_summary.csv"267 if r8.exists():268 _style(); S = pd.read_csv(r8)269 fig, axes = plt.subplots(1, 2, figsize=(8.4, 3.0), sharey=True)270 order = ["router", "best_static_dev", "full_context", "native_cache", "attention_cache_aware", "structured", "oracle_infeasible"]271 col = {"router": "#e87ba4", "best_static_dev": "#2a78d6", "full_context": "#2a78d6", "native_cache": "#6250d6", "attention_cache_aware": "#1baf7a", "structured": "#eda100", "oracle_infeasible": "#7a7a78"}272 for ax, acc in zip(axes, ["blind", "aware"]):273 d = S[S["access"] == acc]274 seen_pts = {}275 for _, r in d.iterrows():276 key = (round(r["cost_ms"]), round(r["quality"], 4))277 lab = r["policy"].replace("_", " ").replace("best static dev", "best static (dev)")278 if key in seen_pts: # identical policies (e.g. dev-best static == full context): one label279 seen_pts[key].set_text(seen_pts[key].get_text() + " = " + lab); continue280 mk = "*" if r["policy"] == "router" else ("x" if r["policy"] == "oracle_infeasible" else "o")281 ax.errorbar(r["cost_ms"], r["quality"], xerr=[[r["cost_ms"] - r["c_lo"]], [r["c_hi"] - r["cost_ms"]]], yerr=[[r["quality"] - r["q_lo"]], [r["q_hi"] - r["quality"]]], fmt=mk, color=col.get(r["policy"], "#444"), ms=8 if mk == "*" else 5, capsize=2, lw=0.9)282 seen_pts[key] = ax.annotate(lab, (r["cost_ms"], r["quality"]), fontsize=6.5, xytext=(4, -9 if r["policy"] == "router" else 3), textcoords="offset points")283 ax.set_xscale("log"); ax.set_xlabel("mean amortized latency per query, ms (log)"); ax.set_title(f"{acc} regime (held-out test, n={int(d['n'].max())})", fontsize=8.5)284 axes[0].set_ylabel("mean quality (all families, both pairs)")285 fig.suptitle("E8: dev-fitted router vs static policies on held-out test instances (95% cluster-bootstrap bars)", fontsize=9)286 _save_fig(fig, "fig7_router", ["results/tables/E8_router_summary.csv", "runs/raw/E1_core.jsonl", "runs/checkpoints/router_policy.json"], "oracle = infeasible upper reference")287 write_json(ROOT / "results/provenance.json", PROV)288 print(f"analysis done: {len(PROV)} artifacts with provenance")289 return 0290