1"""Build every empirical table and figure from raw records. Each output gets a provenance entry (source run files + script)."""2from __future__ import annotations3import json, sys, glob4from pathlib import Path5import numpy as np, pandas as pd6import matplotlib7matplotlib.use("Agg")8import matplotlib.pyplot as plt9from ..util import ROOT, read_json, write_json, read_jsonl, utc_now10from .load import load_experiments, LABELS11from .stats import cluster_bootstrap_mean, paired_difference, holm, summarize_latency1213# fixed hue per method family (colorblind-validated palette; see results/figures/PALETTE.txt)14PAL = {"A": "#2a78d6", "B_quant": "#eb6834", "B_sel": "#1baf7a", "B_ref": "#6250d6", "C": "#e87ba4", "A_struct": "#eda100", "neg": "#e34948", "hosted": "#008300"}15METHOD_COLOR = {"A0": "#7a7a78", "A1": "#2a78d6", "A2": "#2a78d6", "A3": "#2a78d6", "A4": "#2a78d6", "A5": "#eda100", "A6": "#eda100", "A7": "#2a78d6", "A8": "#2a78d6",16 "B0": "#6250d6", "B1": "#6250d6", "B2": "#eb6834", "B3": "#1baf7a", "B4": "#e34948", "B5": "#1baf7a", "B6": "#1baf7a", "B9": "#eb6834", "C0": "#e34948", "C2": "#e87ba4", "C3": "#e87ba4"}17MARKER = {"A0": "x", "A1": "o", "A2": "v", "A3": "^", "A4": "s", "A5": "D", "A6": "P", "A7": "o", "A8": "<", "B0": "o", "B1": "o", "B2": "s", "B3": "^", "B4": "x", "B5": "D", "B6": "P", "B9": "*", "C0": "x", "C2": "s", "C3": "D"}18TAB = ROOT / "results/tables"; FIG = ROOT / "results/figures"19PROV: dict = {}2021def _prov(path: Path, sources: list[str], note: str = ""):22 PROV[str(path.relative_to(ROOT))] = {"sources": sources, "script": "src/wp30/analysis/analyze.py", "generated_utc": utc_now(), "note": note}2324def _save_fig(fig, name: str, sources: list[str], note: str = ""):25 FIG.mkdir(parents=True, exist_ok=True)26 for ext in ("pdf", "png"):27 p = FIG / f"{name}.{ext}"; fig.savefig(p, bbox_inches="tight", dpi=160 if ext == "png" else None)28 _prov(p, sources, note)29 plt.close(fig)3031def _style():32 plt.rcParams.update({"font.size": 9, "axes.spines.top": False, "axes.spines.right": False, "axes.grid": True, "grid.alpha": 0.25, "grid.linewidth": 0.5,33 "legend.frameon": False, "figure.dpi": 110, "axes.titlesize": 9.5, "axes.labelsize": 9})3435def fmt_bytes(b):36 return f"{b/1e6:.1f} MB" if b >= 1e6 else (f"{b/1e3:.1f} kB" if b >= 1e3 else f"{b:.0f} B")3738# ------------------------------------------------------------------------------------------------------------- tables39def table_primary(df: pd.DataFrame, exp: str, sources: list[str]) -> pd.DataFrame:40 rows = []41 for (fam, pair, cond), g in df.groupby(["task_family", "pair", "condition"]):42 q = cluster_bootstrap_mean(g, "quality_operational"); ok = (g["status"] == "success").mean()43 rows.append({"family": fam, "pair": pair, "condition": cond, "method": g["method_id"].iloc[0], "access": g["question_access"].iloc[0], "n": q["n"], "n_docs": q["n_clusters"],44 "quality": q["mean"], "q_lo": q["lo"], "q_hi": q["hi"], "success_rate": ok, "payload_bytes_median": g["serialized_payload_bytes"].median(),45 "cache_bytes_median": g["cache_tensor_bytes"].median(), "cold_ms_p50": g["cost_cold_ms"].median(), "warm_ms_p50": g["cost_warm_query_ms"].median(),46 "amortized_ms_p50": g["cost_amortized_ms"].median(), "ttft_ms_p50": g["ttft_ms"].median(), "peak_decode_mem_median": g["peak_decode_memory"].median(), "peak_cold_mem_median": g["peak_memory_cold"].median(),47 "fetched_bytes_mean": g["fetched_bytes"].mean()})48 t = pd.DataFrame(rows).sort_values(["family", "pair", "condition"])49 TAB.mkdir(parents=True, exist_ok=True); p = TAB / f"{exp}_primary.csv"; t.to_csv(p, index=False); _prov(p, sources, "per-condition quality (cluster bootstrap CI) and resource medians")50 return t5152def table_failures(df: pd.DataFrame, exp: str, sources):53 t = df.groupby(["method_id", "status"]).size().unstack(fill_value=0)54 p = TAB / f"{exp}_failures.csv"; t.to_csv(p); _prov(p, sources, "attempt status counts by method"); return t5556def table_qtype(df: pd.DataFrame, exp: str, sources):57 d = df[df["task_family"] == "synthetic"]58 rows = []59 for (cond, qt), g in d.groupby(["condition", "qtype"]):60 q = cluster_bootstrap_mean(g, "quality_operational", n_boot=500); rows.append({"condition": cond, "qtype": qt, "quality": q["mean"], "lo": q["lo"], "hi": q["hi"], "n": q["n"]})61 t = pd.DataFrame(rows); p = TAB / f"{exp}_by_question_type.csv"; t.to_csv(p, index=False); _prov(p, sources, "synthetic accuracy by question type"); return t6263def primary_tests(df: pd.DataFrame, exp: str, sources, tests: list[dict]):64 rows = []65 for t in tests:66 sub = df[df["task_family"].isin(t["families"])] if t.get("families") else df67 if t.get("pair"): sub = sub[sub["pair"] == t["pair"]]68 r = paired_difference(sub, t.get("value", "quality_operational"), "condition", t["a"], t["b"], keys=("document_id", "question_id", "pair"))69 r.update({"name": t["name"], "hypothesis": t.get("hypothesis", "")}); rows.append(r)70 out = pd.DataFrame(rows)71 if len(out):72 out["p_holm"] = holm([p if not np.isnan(p) else 1.0 for p in out["p"]])73 p = TAB / f"{exp}_primary_tests.csv"; out.to_csv(p, index=False); _prov(p, sources, "registered primary comparisons, paired, cluster bootstrap, Holm"); return out7475# ------------------------------------------------------------------------------------------------------------ figures76def _short_pair(p: str) -> str:77 return p.replace("Qwen2.5-3B-Instruct-bf16->Qwen2.5-3B-Instruct-bf16", "Qwen2.5-3B").replace("Llama-3.2-3B-Instruct-bf16->Llama-3.2-3B-Instruct-bf16", "Llama-3.2-3B").replace("Qwen2.5-1.5B-Instruct-bf16->Qwen2.5-3B-Instruct-bf16", "Qwen1.5B->3B")7879def _mk(m: str, bits) -> str:80 if m == "B2" and bits == 4: return "p"81 if m == "A7": return "h"82 return MARKER.get(m, "o")8384def fig_quality_vs_bytes(df: pd.DataFrame, exp: str, sources, access_filter=None, name="fig2_quality_vs_bytes"):85 _style()86 fams = [f for f in ["synthetic", "exec", "public_qasper", "public_2wikimqa"] if f in set(df["task_family"])]87 pairs = sorted(df["pair"].unique())88 fig, axes = plt.subplots(len(pairs), len(fams), figsize=(3.1 * len(fams), 2.7 * len(pairs)), squeeze=False, sharey="row")89 for i, pair in enumerate(pairs):90 for j, fam in enumerate(fams):91 ax = axes[i][j]; d = df[(df["pair"] == pair) & (df["task_family"] == fam)]92 if access_filter: d = d[d["question_access"].isin(access_filter)]93 for cond, g in d.groupby("condition"):94 m = g["method_id"].iloc[0]; q = cluster_bootstrap_mean(g, "quality_operational", n_boot=300)95 x = g["serialized_payload_bytes"].median()96 if m == "A0" or x is None or np.isnan(x): continue97 ax.errorbar(max(x, 1), q["mean"], yerr=[[q["mean"] - q["lo"]], [q["hi"] - q["mean"]]], fmt=_mk(m, g["bits"].iloc[0]), color=METHOD_COLOR.get(m, "#444"), ms=4.5, lw=0.9, capsize=1.5, mfc="white" if m.startswith("B") else None)98 ax.set_xscale("log"); ax.set_title(f"{fam.replace('public_', '')} - {_short_pair(pair)}", fontsize=8.5); ax.set_ylim(-0.03, 1.03)99 if i == len(pairs) - 1: ax.set_xlabel("serialized payload bytes (log)")100 if j == 0: ax.set_ylabel("quality (acc. or F1)")101 present = set(df["method_id"]); has4 = ((df["method_id"] == "B2") & (df["bits"] == 4)).any()102 handles = [plt.Line2D([], [], marker=_mk(m, None), color=METHOD_COLOR[m], ls="", ms=5, mfc="white" if m.startswith("B") else None, label=LABELS[m] + (" 8-bit" if m == "B2" else "")) for m in ["A1", "A2", "A3", "A4", "A5", "A6", "A7", "A8", "B0", "B1", "B2", "B3", "B4", "B5", "B6", "B9", "C0", "C2", "C3"] if m in present]103 if has4: handles.append(plt.Line2D([], [], marker="p", color=METHOD_COLOR["B2"], ls="", ms=5, mfc="white", label="B2 packed quant 4-bit"))104 fig.legend(handles=handles, loc="lower center", ncol=5, fontsize=7, bbox_to_anchor=(0.5, -0.06 - 0.02 * (len(handles) // 5)))105 fig.subplots_adjust(hspace=0.5); fig.suptitle("Quality vs measured serialized payload bytes (medians; 95% cluster-bootstrap CIs)", fontsize=9.5, y=1.0)106 _save_fig(fig, name, sources, "points = conditions; x = median payload bytes; y = mean quality")107108def fig_quality_vs_latency(df: pd.DataFrame, exp: str, sources, name="fig3_quality_vs_latency_memory"):109 _style()110 from matplotlib.ticker import NullFormatter111 df = df[(df["hardware_id"] == "laptop-m5max-48g") & (df["pair"].str.startswith("Qwen"))] # one machine, one pair: comparable timings112 fams = [f for f in ["synthetic", "exec", "public_qasper", "public_2wikimqa"] if f in set(df["task_family"])]113 fig, axes = plt.subplots(2, len(fams), figsize=(3.1 * len(fams), 5.6), squeeze=False)114 for j, fam in enumerate(fams):115 d = df[df["task_family"] == fam]116 for cond, g in d.groupby("condition"):117 m = g["method_id"].iloc[0]; q = g["quality_operational"].mean()118 if m == "A0": continue119 axes[0][j].scatter(g["cost_cold_ms"].median(), q, marker=_mk(m, g["bits"].iloc[0]), color=METHOD_COLOR.get(m, "#444"), s=22, facecolors="white" if m.startswith("B") else METHOD_COLOR.get(m, "#444"), lw=0.9)120 axes[1][j].scatter(g["peak_memory_cold"].median() / 1e9, q, marker=_mk(m, g["bits"].iloc[0]), color=METHOD_COLOR.get(m, "#444"), s=22, facecolors="white" if m.startswith("B") else METHOD_COLOR.get(m, "#444"), lw=0.9)121 axes[0][j].set_xscale("log"); axes[0][j].xaxis.set_minor_formatter(NullFormatter()); axes[0][j].set_title(fam.replace("public_", ""), fontsize=8.5); axes[0][j].set_xlabel("cold latency, ms (log)"); axes[1][j].set_xlabel("peak MLX memory incl. build, GB")122 axes[0][j].set_ylim(-0.03, 1.03); axes[1][j].set_ylim(-0.03, 1.03)123 axes[0][0].set_ylabel("quality"); axes[1][0].set_ylabel("quality")124 present = set(df["method_id"])125 handles = [plt.Line2D([], [], marker=_mk(m, None), color=METHOD_COLOR[m], ls="", ms=5, mfc="white" if m.startswith("B") else None, label=LABELS[m] + (" 8-bit" if m == "B2" else "")) for m in ["A1", "A2", "A3", "A4", "A5", "A6", "A7", "A8", "B0", "B1", "B2", "B3", "B4", "B5", "B6", "B9"] if m in present]126 if ((df["method_id"] == "B2") & (df["bits"] == 4)).any(): handles.append(plt.Line2D([], [], marker="p", color=METHOD_COLOR["B2"], ls="", ms=5, mfc="white", label="B2 packed quant 4-bit"))127 fig.legend(handles=handles, loc="lower center", ncol=6, fontsize=7, bbox_to_anchor=(0.5, -0.07))128 fig.subplots_adjust(hspace=0.45)129 fig.suptitle("Quality vs cold latency (top) and peak MLX memory (bottom) - Qwen2.5-3B pair on the laptop (models resident: ~6.5 GB baseline)", fontsize=9)130 _save_fig(fig, name, sources)131132def fig_latency_decomposition(df: pd.DataFrame, exp: str, sources, name="fig4_latency_decomposition"):133 _style()134 d = df[(df["task_family"] == "synthetic") & (df["status"] == "success") & (df["hardware_id"] == "laptop-m5max-48g") & (df["pair"].str.startswith("Qwen"))]135 conds = [c for c in ["A1", "A3|budget_tokens=512|aware", "A4|budget_tokens=512|blind", "A5|budget_tokens=512|blind", "A7|codec=zstd", "A8|budget_tokens=512|blind", "B0", "B1", "B2|bits=8", "B3|keep_frac=0.25", "B5|keep_frac=0.25|aware", "B6|keep_frac=0.25|aware"] if c in set(d["condition"])]136 comps = [("span_encode_ms", "sender encode (text)"), ("span_doc_prefill_ms", "document prefill (cache)"), ("span_compress_ms", "compress/select"), ("span_serialize_ms", "serialize"), ("span_load_ms", "deserialize/load"), ("span_receiver_prefill_ms", "receiver prefill"), ("ttft_ms", "query prefill→first token"), ("span_decode_ms", "decode")]137 cols = ["#2a78d6", "#6250d6", "#1baf7a", "#eda100", "#e87ba4", "#eb6834", "#008300", "#7a7a78"]138 fig, ax = plt.subplots(figsize=(7.2, 3.8))139 for i, c in enumerate(conds):140 g = d[d["condition"] == c]; left = 0141 for (col, lab), color in zip(comps, cols):142 if col == "span_encode_ms" and c.startswith("B"):143 continue # for cache methods the encode span is the sum of the components plotted separately144 v = g[col].median() if col in g and g[col].notna().any() else 0.0145 if v and v > 0:146 ax.barh(i, v, left=left, color=color, height=0.6, edgecolor="white", linewidth=0.8); left += v147 ax.set_yticks(range(len(conds))); ax.set_yticklabels([c.replace("|", " ") for c in conds], fontsize=7.5); ax.invert_yaxis(); ax.set_xlabel("median span duration, ms (serial accounting; one question per document build)")148 ax.legend(handles=[plt.Rectangle((0, 0), 1, 1, color=c) for c in cols], labels=[lab for _, lab in comps], fontsize=7, ncol=4, loc="upper center", bbox_to_anchor=(0.5, -0.22)); ax.set_title("Latency decomposition - synthetic family, Qwen2.5-3B on the laptop, cold scenario (document build counted once)", fontsize=8.5)149 _save_fig(fig, name, sources)150151def fig_handoffs(df: pd.DataFrame, exp: str, sources, name="fig5_handoffs"):152 _style()153 d = df[df["topology"].astype(str).str.startswith("chain")]154 if d.empty: return155 fig, axes = plt.subplots(1, 3, figsize=(9.6, 3.4))156 d["chain"] = d["run_group_id"].str.split(":").str[1]157 for chain, g in d.groupby("chain"):158 rep = g["method_id"].iloc[0]; mode = g["topology"].iloc[0].replace("chain_", ""); agents = chain.split("|")[-1]159 alt = "-" in agents160 color = "#eda100" if rep == "A5" else "#2a78d6"; ls = "-" if mode == "closed" else "--"; mk = "s" if alt else "o"161 hop = g.groupby("handoff_index")162 ret = hop["retention_rate"].mean(); uns = hop["unsupported"].mean(); acc = hop["quality_operational"].mean()163 lab = f"{rep} {mode}" + (" (Qwen/Llama alternating)" if alt else " (Qwen)")164 axes[0].plot(ret.index, ret.values, ls, color=color, marker=mk, ms=4, label=lab, mfc="white" if alt else None)165 axes[1].plot(uns.index, uns.values, ls, color=color, marker=mk, ms=4, mfc="white" if alt else None)166 axes[2].plot(acc.index, acc.values, ls, color=color, marker=mk, ms=4, mfc="white" if alt else None)167 for ax, t in zip(axes, ["retained ledger facts (share of 62)", "unsupported numbers/ids per message", "answer accuracy (15 question types)"]):168 ax.set_xlabel("handoffs"); ax.set_title(t, fontsize=8.5); ax.set_xticks(sorted(d["handoff_index"].unique()))169 axes[0].set_ylim(0, 1.02); axes[2].set_ylim(0, 1.02); axes[1].set_ylim(0, 0.3)170 fig.subplots_adjust(bottom=0.3, wspace=0.3)171 fig.legend(*axes[0].get_legend_handles_labels(), loc="lower center", ncol=3, fontsize=7, bbox_to_anchor=(0.5, 0.0))172 _save_fig(fig, name, sources, "means over documents; chains defined in configs/chains*.yaml")173174def fig_breakeven(net_rows: list[dict], exp: str, sources, name="fig6_breakeven"):175 """Left: measured two-node transfer vs remote text prefill (medians with p25-p75 bars per target length).176 Right: analytic transfer-time model from the measured bandwidth and the measured remote prefill slope."""177 _style()178 if not net_rows: return179 n = pd.DataFrame(net_rows); n = n[n["label"] != "loopback"]180 fig, axes = plt.subplots(1, 2, figsize=(8.6, 3.1))181 ax = axes[0]182 series = [("text_remote_prefill", None, "none", "text -> remote prefill", "#2a78d6", "o", "-"),183 ("B1", None, "none", "B1 bf16 cache", "#6250d6", "D", "-"), ("B1", None, "zstd", "B1 bf16 + zstd", "#6250d6", "D", ":"),184 ("B2", 8.0, "none", "B2 8-bit cache", "#eb6834", "s", "-"), ("B2", 4.0, "none", "B2 4-bit cache (Qwen: quality 0)", "#e34948", "x", "--")]185 for meth, bits, codec, lab, color, mk, ls in series:186 g = n[(n["method"] == meth) & (n["codec"].fillna("none") == codec)]187 g = g[g["bits"].isna()] if bits is None else g[g["bits"] == bits]188 if g.empty: continue189 agg = g.groupby("target_tokens").agg(x=("prefix_tokens", "median"), y=("roundtrip_ms", "median"), lo=("roundtrip_ms", lambda s: s.quantile(0.25)), hi=("roundtrip_ms", lambda s: s.quantile(0.75))).reset_index()190 ax.errorbar(agg["x"], agg["y"], yerr=[agg["y"] - agg["lo"], agg["hi"] - agg["y"]], fmt=mk + ls, color=color, ms=4.5, lw=1.1, capsize=2, label=lab, mfc="white" if meth == "B1" else None)191 loc = n[n["method"] == "local_prefill"]192 if len(loc):193 agg = loc.groupby("target_tokens").agg(x=("prefix_tokens", "median"), y=("local_prefill_ms", "median")).reset_index()194 ax.plot(agg["x"], agg["y"], "^-.", color="#7a7a78", ms=4.5, lw=1, label="local prefill only (no transfer, no decode)")195 ax.set_xlabel("context tokens (median per length class)"); ax.set_ylabel("ms"); ax.set_title("Measured: M3U96a -> M3U96b, 1 GbE (18 runs per point)", fontsize=8.5)196 ax.legend(fontsize=6.5, loc="upper left")197 ax = axes[1]198 bw = n[n["method"].isin(["B1", "B2"])]["effective_gbps"].median()199 b1 = n[(n["method"] == "B1") & (n["codec"].fillna("none") == "none")]200 bpt = (b1["serialized_bytes"] / b1["prefix_tokens"]).median() if len(b1) else np.nan201 tr = n[n["method"] == "text_remote_prefill"]202 if len(tr) and not np.isnan(bpt):203 slope = (tr["remote_prefill_ms"] / tr["prefix_tokens"]).median()204 L = np.linspace(500, 32000, 50)205 for frac, lab, color in [(1.0, "bf16 cache", "#6250d6"), (0.53, "8-bit cache", "#eb6834")]:206 for bw_g, ls, alpha in [(bw, "-", 0.95), (10.0, "--", 0.5)]:207 ax.plot(L, L * bpt * frac * 8 / (bw_g * 1e9) * 1000, ls, color=color, lw=1.1, alpha=alpha, label=f"{lab} transfer @ {bw_g:.1f} Gb/s" + (" (hypothetical)" if bw_g == 10.0 else " (measured)"))208 ax.plot(L, L * slope, "-", color="#2a78d6", lw=1.6, label="remote text prefill (measured slope, M3 Ultra)")209 ax.set_xlabel("context tokens"); ax.set_ylabel("ms"); ax.set_title("Analytic model (Qwen2.5-3B cache, measured slopes)", fontsize=8.5)210 ax.legend(fontsize=6.3, loc="upper left"); ax.set_ylim(0, None)211 _save_fig(fig, name, sources, "left = measured (medians, p25-p75); right = analytic model from measured bandwidth and prefill slope; dashed 10 Gb/s is hypothetical")212213# ------------------------------------------------------------------------------------------------------------ main214def main(protocol_path: str) -> int:215 _style()216 prot = read_json(ROOT / protocol_path) if (ROOT / protocol_path).exists() else {}217 core_exp = prot.get("core_experiment_ids", ["E1_core"]) if prot else ["E1_core"]218 exps = {p.stem: p.name for p in (ROOT / "runs/raw").glob("*.jsonl")}219 (FIG).mkdir(parents=True, exist_ok=True); (TAB).mkdir(parents=True, exist_ok=True)220 (FIG / "PALETTE.txt").write_text("categorical palette validated with dataviz validate_palette.js (light): #2a78d6,#eb6834,#1baf7a,#eda100,#e87ba4,#008300,#6250d6,#e34948 — ALL CHECKS PASS\n")221 # choose the experiment set to analyze: core if present, else pilot222 main_exp = "E1_core" if "E1_core" in exps else ("E1_pilot" if "E1_pilot" in exps else None)223 if main_exp:224 src = [f"runs/raw/{main_exp}.jsonl"]225 df = load_experiments([f"{main_exp}.jsonl"]); df_all = load_experiments([f"{main_exp}.jsonl"], last_attempt_only=False)226 table_primary(df, main_exp, src); table_failures(df_all, main_exp, src); table_qtype(df, main_exp, src)227 tests = [228 {"name": "T1 A5 vs A4 (512, blind, synthetic)", "a": "A5|budget_tokens=512|blind", "b": "A4|budget_tokens=512|blind", "families": ["synthetic"], "hypothesis": "RQ5/H1"},229 {"name": "T2 B2-8bit vs B0", "a": "B2|bits=8", "b": "B0", "hypothesis": "H2"},230 {"name": "T3 B2-4bit vs B0", "a": "B2|bits=4", "b": "B0", "hypothesis": "H2"},231 {"name": "T4 B5-aware vs B3 (1/4)", "a": "B5|keep_frac=0.25|aware", "b": "B3|keep_frac=0.25", "hypothesis": "H3"},232 {"name": "T6 A1 vs B0 (text vs native cache, same content)", "a": "A1", "b": "B0", "hypothesis": "consistency"},233 ]234 primary_tests(df, main_exp, src, tests)235 fig_quality_vs_bytes(df, main_exp, src)236 fig_quality_vs_bytes(df[df["question_access"].isin(["blind", "none"])], main_exp, src, name="figA_quality_vs_bytes_blind")237 fig_quality_vs_bytes(df[df["question_access"] == "aware"], main_exp, src, name="figA_quality_vs_bytes_aware")238 fig_quality_vs_latency(df, main_exp, src); fig_latency_decomposition(df, main_exp, src)239 chain_exp = "E4_core" if "E4_core" in exps else ("E4_pilot" if "E4_pilot" in exps else None)240 if chain_exp:241 dfc = load_experiments([f"{chain_exp}.jsonl"]); src = [f"runs/raw/{chain_exp}.jsonl"]242 from .chains_recompute import recompute243 dfc = recompute(dfc, chain_exp) # ledger metrics recomputed from stored hop messages (span-offset fix)244 fig_handoffs(dfc, chain_exp, src)245 rows = []246 for (grp, h), g in dfc.groupby(["run_group_id", "handoff_index"]):247 q = cluster_bootstrap_mean(g, "quality_operational", n_boot=500); r = cluster_bootstrap_mean(g.drop_duplicates("document_id"), "retention_rate", n_boot=500)248 rows.append({"chain": grp.split(":")[1], "handoffs": h, "accuracy": q["mean"], "acc_lo": q["lo"], "acc_hi": q["hi"], "retention": r["mean"], "ret_lo": r["lo"], "ret_hi": r["hi"],249 "unsupported_mean": g.drop_duplicates("document_id")["unsupported"].mean(), "fetched_bytes_mean": g["fetched_bytes"].mean(), "msg_bytes_mean": g["msg_bytes_hop"].mean(), "n": q["n"], "n_docs": q["n_clusters"]})250 t = pd.DataFrame(rows); p = TAB / f"{chain_exp}_handoffs.csv"; t.to_csv(p, index=False); _prov(p, src, "repeated handoffs")251 primary_tests(dfc[dfc["handoff_index"] == 4].assign(condition=dfc["run_group_id"].str.split(":").str[1]), chain_exp + "_h4", src,252 [{"name": "T5 A5 chain vs A4 chain at 4 handoffs (closed)", "a": "A5|closed|qwen3b", "b": "A4|closed|qwen3b", "hypothesis": "H5"}])253 fan_exp = "E5_core" if "E5_core" in exps else ("E5_pilot" if "E5_pilot" in exps else None)254 if fan_exp:255 dff = load_experiments([f"{fan_exp}.jsonl"]); src = [f"runs/raw/{fan_exp}.jsonl"]256 rows = []257 for (cond, qt), g in dff.groupby(["condition", "qtype"]):258 q = cluster_bootstrap_mean(g, "quality_operational", n_boot=500); rows.append({"condition": cond, "qtype": qt, "quality": q["mean"], "lo": q["lo"], "hi": q["hi"], "n": q["n"], "payload_bytes_median": g["serialized_payload_bytes"].median(), "fetched_bytes_mean": g["fetched_bytes"].mean()})259 t = pd.DataFrame(rows); p = TAB / f"{fan_exp}_fanin.csv"; t.to_csv(p, index=False); _prov(p, src, "fan-in by method and question type")260 net_files = [p for p in (ROOT / "runs/raw").glob("E6_*.jsonl") if "smoke" not in p.name] or list((ROOT / "runs/raw").glob("E6_*.jsonl"))261 if net_files:262 rows = [r for p in net_files for r in read_jsonl(p)]263 fig_breakeven(rows, "E6", [str(p.relative_to(ROOT)) for p in net_files])264 n = pd.DataFrame(rows)265 agg = n.groupby(["label", "method", "bits", "codec", "target_tokens"], dropna=False).agg(n=("repeat", "size"), bytes=("serialized_bytes", "median"), roundtrip_ms_p50=("roundtrip_ms", "median"), send_ms_p50=("send_ms", "median"),266 remote_load_ms_p50=("remote_load_ms", "median"), remote_ttft_p50=("remote_ttft_ms", "median"), remote_prefill_p50=("remote_prefill_ms", "median"), eff_gbps=("effective_gbps", "median"), correct=("correct", "mean"),267 local_prefill_p50=("local_prefill_ms", "median"), local_ttft_p50=("local_ttft_ms", "median"), remote_decode_p50=("remote_decode_ms", "median")).reset_index()268 p = TAB / "E6_network.csv"; agg.to_csv(p, index=False); _prov(p, [str(x.relative_to(ROOT)) for x in net_files], "two-node transfer measurements")269 hosted = [p for p in (ROOT / "runs/raw").glob("ED_hosted_core.jsonl")] or [p for p in (ROOT / "runs/raw").glob("ED_hosted_pilot.jsonl")]270 if hosted:271 dh = load_experiments([p.name for p in hosted]); src = [str(p.relative_to(ROOT)) for p in hosted]272 rows = []273 for (fam, recv, cond), g in dh.groupby(["task_family", "receiver", "condition"]):274 q = cluster_bootstrap_mean(g, "quality_operational", n_boot=500)275 rows.append({"family": fam, "provider_model": recv, "condition": cond, "quality": q["mean"], "lo": q["lo"], "hi": q["hi"], "n": q["n"], "cost_usd_mean": g["cost_usd"].mean(), "ttft_ms_p50": g["ttft_ms"].median(), "total_ms_p50": g["total_wall_ms"].median(), "success_rate": (g["status"] == "success").mean()})276 t = pd.DataFrame(rows); p = TAB / "ED_hosted.csv"; t.to_csv(p, index=False); _prov(p, src, "hosted receivers; cost estimated from provider price tables")277 if "E3_core" in exps:278 d3 = load_experiments(["E3_core.jsonl"]); s3 = load_experiments(["E3_core_sender.jsonl"]) if "E3_core_sender" in exps else pd.DataFrame()279 src3 = ["runs/raw/E3_core.jsonl", "runs/raw/E3_core_sender.jsonl"]280 d3["cond_s"] = d3["condition"].str.replace("|mapper=runs/checkpoints/mapper_v2_qwen1b5_to_qwen3b.npz", "", regex=False).str.replace("budget_tokens=512", "512", regex=False).str.replace("keep_frac=0.25", "1/4", regex=False)281 if len(s3): s3["cond_s"] = "sender alone " + s3["condition"]282 allr = pd.concat([d3, s3], ignore_index=True)283 rows = []284 for (fam, cond), g in allr.groupby(["task_family", "cond_s"]):285 q = cluster_bootstrap_mean(g, "quality_operational"); byq = g.groupby("qtype")["quality_operational"].mean().to_dict()286 rows.append({"family": fam, "condition": cond, "quality": q["mean"], "lo": q["lo"], "hi": q["hi"], "n": q["n"], "n_docs": q["n_clusters"], "payload_bytes_median": g["serialized_payload_bytes"].median(),287 "cold_ms_p50": g["cost_cold_ms"].median(), "warm_ms_p50": g["cost_warm_query_ms"].median(), "unknown_share": g["prediction_artifact"].fillna("").str.lower().str.contains("unknown").mean(), **{f"q_{k}": v for k, v in byq.items()}})288 t3 = pd.DataFrame(rows); p = TAB / "E3_core_transfer.csv"; t3.to_csv(p, index=False); _prov(p, src3, "same-family mapper transfer vs text handoffs and references")289 primary_tests(d3.assign(condition=d3["cond_s"]), "E3_core", src3, [{"name": "C2 mapper vs A4 summary (synthetic)", "a": "C2", "b": "A4|512|blind", "families": ["synthetic"], "hypothesis": "H10"},290 {"name": "C2 mapper vs A1 receiver full context (synthetic)", "a": "C2", "b": "A1", "families": ["synthetic"], "hypothesis": "H10"}])291 if "E7_qtypes" in exps:292 d7 = load_experiments(["E7_qtypes.jsonl"]); src7 = ["runs/raw/E7_qtypes.jsonl"]293 rows = []294 for (cond, qt, pair), g in d7.groupby(["condition", "qtype", "pair"]):295 q = cluster_bootstrap_mean(g, "quality_operational", n_boot=500); rows.append({"condition": cond, "qtype": qt, "pair": _short_pair(pair), "quality": q["mean"], "lo": q["lo"], "hi": q["hi"], "n": q["n"], "abstained": g["abstained"].mean()})296 t7 = pd.DataFrame(rows); p = TAB / "E7_qtypes.csv"; t7.to_csv(p, index=False); _prov(p, src7, "EXPLORATORY: question types 6-15 on synthetic test documents")297 _style(); fig, axes = plt.subplots(1, 2, figsize=(9.4, 3.4), sharey=True)298 qorder = ["negation", "revision", "temporal", "aggregation", "unit_conversion", "unanswerable"]299 for ax, pair in zip(axes, sorted(t7["pair"].unique())):300 d = t7[t7["pair"] == pair]301 conds = [c for c in ["A1", "A3|budget_tokens=512|blind", "A4|budget_tokens=512|blind", "A5|budget_tokens=512|blind", "B2|bits=8", "B3|keep_frac=0.25", "B5|keep_frac=0.25|aware"] if c in set(d["condition"])]302 w = 0.8 / max(1, len(conds)); x = np.arange(len(qorder))303 for i, c in enumerate(conds):304 m = c.split("|")[0]; vals = [d[(d["condition"] == c) & (d["qtype"] == qt)]["quality"].mean() if ((d["condition"] == c) & (d["qtype"] == qt)).any() else np.nan for qt in qorder]305 ax.bar(x + (i - len(conds) / 2 + 0.5) * w, vals, w, color=METHOD_COLOR.get(m, "#444"), alpha=0.95 if c in ("A1", "A4|budget_tokens=512|blind", "A5|budget_tokens=512|blind") else 0.6, edgecolor="white", linewidth=0.5, label=LABELS.get(m, m) + (" 8-bit" if m == "B2" else "") + (" (aware)" if "aware" in c else ""))306 ax.set_xticks(x); ax.set_xticklabels([q.replace("_", "\n") for q in qorder], fontsize=7.5); ax.set_title(f"{pair}", fontsize=8.5); ax.set_ylim(0, 1.02)307 axes[0].set_ylabel("accuracy"); axes[0].legend(fontsize=6.5, ncol=2, loc="upper center", bbox_to_anchor=(1.05, -0.2))308 fig.suptitle("EXPLORATORY E7: question types not covered by the core (synthetic test, 40 documents)", fontsize=9)309 _save_fig(fig, "figA_qtypes_exploratory", src7, "exploratory extension A-004")310 r8 = TAB / "E8_router_summary.csv"311 if r8.exists():312 _style(); S = pd.read_csv(r8)313 fig, axes = plt.subplots(1, 2, figsize=(8.4, 3.0), sharey=True)314 order = ["router", "best_static_dev", "full_context", "native_cache", "attention_cache_aware", "structured", "oracle_infeasible"]315 col = {"router": "#e87ba4", "best_static_dev": "#2a78d6", "full_context": "#2a78d6", "native_cache": "#6250d6", "attention_cache_aware": "#1baf7a", "structured": "#eda100", "oracle_infeasible": "#7a7a78"}316 for ax, acc in zip(axes, ["blind", "aware"]):317 d = S[S["access"] == acc]318 seen_pts = {}319 for _, r in d.iterrows():320 key = (round(r["cost_ms"]), round(r["quality"], 4))321 lab = r["policy"].replace("_", " ").replace("best static dev", "best static (dev)")322 if key in seen_pts: # identical policies (e.g. dev-best static == full context): one label323 seen_pts[key].set_text(seen_pts[key].get_text() + " = " + lab); continue324 mk = "*" if r["policy"] == "router" else ("x" if r["policy"] == "oracle_infeasible" else "o")325 ax.errorbar(r["cost_ms"], r["quality"], xerr=[[r["cost_ms"] - r["c_lo"]], [r["c_hi"] - r["cost_ms"]]], yerr=[[r["quality"] - r["q_lo"]], [r["q_hi"] - r["quality"]]], fmt=mk, color=col.get(r["policy"], "#444"), ms=8 if mk == "*" else 5, capsize=2, lw=0.9)326 seen_pts[key] = ax.annotate(lab, (r["cost_ms"], r["quality"]), fontsize=6.5, xytext=(4, -9 if r["policy"] == "router" else 3), textcoords="offset points")327 ax.set_xscale("log"); ax.set_xlabel("mean amortized latency per query, ms (log)"); ax.set_title(f"{acc} regime (held-out test, n={int(d['n'].max())})", fontsize=8.5)328 axes[0].set_ylabel("mean quality (all families, both pairs)")329 fig.suptitle("E8: dev-fitted router vs static policies on held-out test instances (95% cluster-bootstrap bars)", fontsize=9)330 _save_fig(fig, "fig7_router", ["results/tables/E8_router_summary.csv", "runs/raw/E1_core.jsonl", "runs/checkpoints/router_policy.json"], "oracle = infeasible upper reference")331 write_json(ROOT / "results/provenance.json", PROV)332 print(f"analysis done: {len(PROV)} artifacts with provenance")333 return 0334