1#!/usr/bin/env python32"""Preregistered analysis of experiments/outputs/<protocol>/{runs,judgments}.jsonl → analysis tables (CSV/LaTeX) + figures. No data = no results."""3import json, sys, itertools4from pathlib import Path5import numpy as np, pandas as pd6from scipy import stats7sys.path.insert(0, "src")8from jof250 import config, viz9from jof250.provenance import stamp10PROTO = sys.argv[1] if len(sys.argv) > 1 else "heldout_v1"11R = config.ROOT; OUT = R / "experiments" / "outputs" / PROTO; AN = R / "experiments" / "analysis"; AN.mkdir(exist_ok=True); TAB = R / "paper" / "tables"; TAB.mkdir(exist_ok=True)12runs = [json.loads(l) for l in (OUT / "runs.jsonl").read_text().splitlines() if l.strip()]; judg = [json.loads(l) for l in (OUT / "judgments.jsonl").read_text().splitlines() if l.strip()] if (OUT / "judgments.jsonl").exists() else []13DIMS = ["clarity", "argument_logic", "contribution_positioning", "methodological_rigor", "fidelity", "usefulness"]14rows = []15for j in judg:16 if not j.get("ok"): continue17 for cond, sc in j["scores"].items():18 rows.append({"task": j["task"], "doc": j["doc"], "rep": j["rep"], "pass": j["pass"], "condition": cond, **{d: sc.get(d) for d in DIMS}, "n_invented_facts": sc.get("n_invented_facts")})19J = pd.DataFrame(rows); Rn = pd.DataFrame([{**{k: r[k] for k in ("task", "doc", "domain", "condition", "rep", "ok")}, **({k: r["metrics"][k] for k in ("n_words", "preserved_rate", "new_numbers", "n_unverifiable_citations", "n_unsupported_claims", "max_overlap_share", "todo_placeholders")} if r.get("ok") else {}), "cost_usd": r.get("cost_usd"), "latency_s": r.get("latency_s")} for r in runs])20res = {"protocol": PROTO, "generated": stamp(), "n_runs": len(Rn), "n_runs_ok": int(Rn.ok.sum()), "n_judgments": len(judg)}21if len(J):22 JA = J[J["pass"] == "A"]; fin = JA[JA.doc != "wp29"]23 def paired(a, b, outcome, df, higher_better=True):24 pa = df[df.condition == a].set_index(["task", "doc", "rep"])[outcome]; pb = df[df.condition == b].set_index(["task", "doc", "rep"])[outcome]25 d = (pa - pb).dropna()26 if len(d) < 3: return {"n": int(len(d))}27 cl = d.reset_index().groupby(["task", "doc"])[outcome].mean(); rng = np.random.default_rng(45)28 boots = [float(rng.choice(cl.values, len(cl)).mean()) for _ in range(2000)]29 try: p = float(stats.wilcoxon(d.values, zero_method="wilcox").pvalue) if (d != 0).any() else 1.030 except Exception: p = None31 x = pa.loc[d.index].values; y = pb.loc[d.index].values; cliff = float(np.mean([np.sign(xi - yi) for xi in x for yi in y]))32 return {"n": int(len(d)), "mean_diff": round(float(d.mean()), 3), "ci95": [round(float(np.percentile(boots, 2.5)), 3), round(float(np.percentile(boots, 97.5)), 3)], "wilcoxon_p": p, "cliffs_delta": round(cliff, 3)}33 H = {"H1_D_vs_B": ("D_profiles_retrieval", "B_strong_prompt", ["clarity", "argument_logic", "contribution_positioning", "usefulness"]), "H2_D_vs_C": ("D_profiles_retrieval", "C_conventional_rag", ["clarity", "argument_logic", "contribution_positioning", "usefulness"]),34 "H3_E_vs_D": ("E_full_engine", "D_profiles_retrieval", ["methodological_rigor", "fidelity", "usefulness"]), "E_vs_A": ("E_full_engine", "A_baseline", DIMS), "E_vs_B": ("E_full_engine", "B_strong_prompt", DIMS)}35 res["hypotheses"] = {}36 for h, (a, b, outs) in H.items():37 tests = {o: paired(a, b, o, fin) for o in outs}; ps = [(o, t["wilcoxon_p"]) for o, t in tests.items() if t.get("wilcoxon_p") is not None]38 m = len(ps); holm = {}39 for rank, (o, p) in enumerate(sorted(ps, key=lambda x: x[1])): holm[o] = min(1.0, p * (m - rank))40 for o in tests:41 if o in holm: tests[o]["holm_p"] = round(holm[o], 4); tests[o]["significant"] = bool(holm[o] < 0.05 and (tests[o]["ci95"][0] > 0 or tests[o]["ci95"][1] < 0))42 res["hypotheses"][h] = tests43 res["ablations"] = {ab: {o: paired("E_full_engine", ab, o, fin) for o in DIMS} for ab in ["E_no_retrieval", "E_no_profile", "E_no_referee", "E_no_factcheck"] if (fin.condition == ab).any()}44 means = fin.groupby("condition")[DIMS + ["n_invented_facts"]].mean().round(2); means["n"] = fin.groupby("condition").size(); means.to_csv(AN / "judge_means_by_condition.csv")45 res["judge_means"] = json.loads(means.to_json(orient="index"))46 res["task_means"] = json.loads(fin.groupby(["task", "condition"])[DIMS].mean().round(2).to_json(orient="index"))47 ood = JA[JA.doc == "wp29"]48 if len(ood): res["out_of_domain_means"] = json.loads(ood.groupby("condition")[DIMS].mean().round(2).to_json(orient="index"))49 # judge consistency50 JB = J[J["pass"] == "B"]51 if len(JB):52 mm = JA.merge(JB, on=["task", "doc", "rep", "condition"], suffixes=("_a", "_b"))53 res["judge_consistency"] = {"n_pairs": int(len(mm)), **{d: {"mean_abs_diff": round(float((mm[d + "_a"] - mm[d + "_b"]).abs().mean()), 3), "spearman": round(float(stats.spearmanr(mm[d + "_a"], mm[d + "_b"]).correlation), 3) if mm[d + "_a"].nunique() > 1 and mm[d + "_b"].nunique() > 1 else None} for d in DIMS}}54 # H4/H5 secondary outcomes55 ok = Rn[Rn.ok & (Rn.domain == "finance")]56 sec = ok.groupby("condition")[["preserved_rate", "new_numbers", "n_unverifiable_citations", "n_unsupported_claims", "max_overlap_share", "todo_placeholders", "n_words", "cost_usd", "latency_s"]].mean().round(3); sec["n"] = ok.groupby("condition").size(); sec.to_csv(AN / "secondary_by_condition.csv")57 res["secondary_means"] = json.loads(sec.to_json(orient="index"))58 res["H4"] = {f"E_vs_{c[0]}": {o: paired("E_full_engine", c, o, ok.assign(rep=ok.rep)) for o in ["n_unverifiable_citations", "n_unsupported_claims", "new_numbers"]} for c in ["A_baseline", "B_strong_prompt", "C_conventional_rag"]}59 # H5: the preregistered metric (max 8-gram overlap with ANY corpus document) counts overlap with the task's own source document60 # (e.g. improve_abstract necessarily reuses the source abstract). Post-hoc refinement, reported as such: recompute excluding the source document.61 from jof250 import auditing62 other = []63 for r in runs:64 if not r.get("ok") or r["domain"] != "finance": continue65 ov = auditing.overlap_with_corpus(r["text"]); oth = [d for d in ov["top_docs"] if d["doc_id"] != r["doc"]]66 other.append({"task": r["task"], "doc": r["doc"], "condition": r["condition"], "rep": r["rep"], "self_share": next((d["share"] for d in ov["top_docs"] if d["doc_id"] == r["doc"]), 0.0), "other_share": oth[0]["share"] if oth else 0.0})67 OV = pd.DataFrame(other); OV.to_csv(AN / "overlap_by_run.csv", index=False)68 res["H5_overlap"] = {"preregistered_metric": {"max_overlap_share_overall": float(ok.max_overlap_share.max()), "share_outputs_below_0.02": float((ok.max_overlap_share < 0.02).mean())},69 "excluding_source_document": {"max_other_share": float(OV.other_share.max()), "share_outputs_below_0.02": float((OV.other_share < 0.02).mean()), "mean_other_share": float(OV.other_share.mean())},70 "self_overlap_by_task": json.loads(OV.groupby("task").self_share.mean().round(3).to_json()), "self_overlap_by_condition": json.loads(OV.groupby("condition").self_share.mean().round(3).to_json())}71 # LaTeX tables72 def tex_table(df, caption, label, fname):73 cols = list(df.columns); L = ["\\begin{table}[htbp]\\centering\\footnotesize", f"\\caption{{{caption}}}\\label{{{label}}}", "\\resizebox{\\linewidth}{!}{\\begin{tabular}{l" + "r" * len(cols) + "}\\toprule", "Condition & " + " & ".join(c.replace("_", " ") for c in cols) + " \\\\\\midrule"]74 L += [f"{idx.replace('_', '\\_')} & " + " & ".join(("" if pd.isna(v) else (f"{v:.2f}" if isinstance(v, float) else str(v))) for v in row) + " \\\\" for idx, row in df.iterrows()]75 L += ["\\bottomrule\\end{tabular}}", "\\begin{minipage}{0.95\\linewidth}\\vspace{2pt}\\scriptsize Notes: generated by scripts/05\\_analyze\\_benchmark.py from recorded runs (protocol heldout\\_v1); judge = claude-sonnet-5-5 blind to condition; scores 1--7.\\end{minipage}", "\\end{table}"]76 (TAB / fname).write_text("\n".join(L) + "\n")77 tex_table(means, "Judge scores by condition (finance held-out documents, pass A)", "tab:judge_means", "tab_judge_means.tex")78 tex_table(sec, "Deterministic secondary outcomes by condition (finance held-out documents)", "tab:secondary", "tab_secondary.tex")79 hyp_rows = []80 for h, tests in res["hypotheses"].items():81 for o, t in tests.items():82 if "mean_diff" in t: hyp_rows.append({"contrast": h, "outcome": o, "n": t["n"], "mean_diff": t["mean_diff"], "ci_low": t["ci95"][0], "ci_high": t["ci95"][1], "holm_p": t.get("holm_p"), "cliffs_delta": t["cliffs_delta"]})83 hd = pd.DataFrame(hyp_rows); hd.to_csv(AN / "hypothesis_tests.csv", index=False)84 if len(hd): tex_table(hd.set_index(hd.contrast + " / " + hd.outcome).drop(columns=["contrast", "outcome"]), "Preregistered paired contrasts (cluster bootstrap 95\\% CI, Holm-adjusted Wilcoxon p)", "tab:hypotheses", "tab_hypotheses.tex")85 # figure86 viz.style(); import matplotlib.pyplot as plt87 conds = [c for c in ["A_baseline", "B_strong_prompt", "C_conventional_rag", "D_profiles_retrieval", "E_full_engine"] if c in means.index]88 fig, ax = plt.subplots(figsize=(7, 3)); w = 0.8 / len(DIMS)89 for i, d in enumerate(DIMS):90 vals = [means.loc[c, d] for c in conds]; se = [fin[fin.condition == c][d].std() / np.sqrt(max((fin.condition == c).sum(), 1)) for c in conds]91 ax.bar(np.arange(len(conds)) + i * w, vals, w, yerr=[1.96 * s for s in se], label=d, color=viz.PALETTE[i], edgecolor=viz.SURFACE, linewidth=0.8, error_kw={"elinewidth": 0.8, "ecolor": viz.TEXT2})92 ax.set_xticks(np.arange(len(conds)) + w * (len(DIMS) - 1) / 2); ax.set_xticklabels([c.split("_")[0] for c in conds]); ax.set_ylim(1, 7); ax.set_ylabel("Judge score (1–7)"); ax.legend(fontsize=6, ncol=3); ax.set_title("Blind judge scores by condition (finance held-out tasks; ±1.96 SE)", fontsize=9)93 fig.tight_layout(); fig.savefig(R / "paper" / "figures" / "fig_judge_by_condition.pdf"); fig.savefig(R / "figures" / "fig_judge_by_condition.pdf"); plt.close(fig)94(AN / f"results_{PROTO}.json").write_text(json.dumps(res, indent=1, default=str)); print(json.dumps({k: v for k, v in res.items() if k in ("n_runs", "n_runs_ok", "n_judgments", "hypotheses", "judge_means", "H5_overlap", "judge_consistency")}, indent=1, default=str)[:6000])95