Repository Public Research preview
1#!/usr/bin/env python32"""Tier-1/2 confirmatory analysis under the frozen selection rule (PREREGISTRATION, D11).34Usage: tier1_analysis.py --dev <dev run id> --conf <confirmatory run id> [--out name]5Eligibility: recipient own-vector dev accuracy >= 0.5. Selected condition: best dev transfer condition (ties:6procrustes, lower layer, alpha 1, add before replace). Contrast: selected minus best control on confirmatory7queries (paired, item bootstrap). H6 per pair: lower bound > 0.10 for >= 2 eligible tasks.8"""9from __future__ import annotations1011import argparse12import json13import sys14from pathlib import Path1516import numpy as np1718ROOT = Path(__file__).resolve().parents[1]192021def is_control(name: str, amended: bool = True) -> bool:22 """Null controls (amended rule A1): random norm-matched, unrelated-task vector, sham map, donor zero-shot replace.23 With amended=False the shuffled-query donor state is also counted as a control (pre-amendment rule)."""24 if "shuffled_query" in name:25 return not amended26 return name.startswith("ctl") or "random_orth" in name or "sham_map" in name272829def rank_key(name: str):30 fam = name.split("|")[0]31 lay = [int(p[1:]) for p in name.split("|") if p.startswith("L") and p[1:].isdigit()]32 alpha = [float(p[1:]) for p in name.split("|") if p.startswith("a") and p[1:].replace(".", "").isdigit()]33 return (0 if "procrustes" in fam else 1, lay[0] if lay else 0, abs((alpha[0] if alpha else 1.0) - 1.0), 1 if fam.startswith("replace") else 0)343536def main() -> None:37 ap = argparse.ArgumentParser()38 ap.add_argument("--dev", required=True)39 ap.add_argument("--conf", required=True)40 ap.add_argument("--out", default="TIER1_CONFIRMATORY.md")41 a = ap.parse_args()42 dev = json.loads((ROOT / "experiments" / "summaries" / a.dev / "results.json").read_text())43 conf = json.loads((ROOT / "experiments" / "summaries" / a.conf / "results.json").read_text())44 out = {"dev": a.dev, "conf": a.conf, "pairs": {}}45 L = [f"# Tier-1/2 confirmatory analysis — dev `{a.dev}`, confirmatory `{a.conf}`", "", "| pair | task | eligible (own vector dev) | selected condition (dev acc) | conf: zero / few / own / selected / best control | selected − best control [95% CI] |", "|---|---|---|---|---|---|"]46 for p, pr in conf["pairs"].items():47 if p not in dev["pairs"]:48 continue49 rows, n_ok = {}, 050 for tn, trc in pr["tasks"].items():51 trd = dev["pairs"][p]["tasks"][tn]52 elig = trd["recipient_own_vector"] >= 0.553 cands = [(k, v) for k, v in trd["conditions"].items() if not is_control(k) and "shuffled_query" not in k]54 sel, selv = max(cands, key=lambda kv: (kv[1]["accuracy"], tuple(-x for x in rank_key(kv[0]))))55 cc = trc["conditions"]56 if sel not in cc:57 rows[tn] = {"eligible": elig, "selected": sel, "error": "selected condition absent in confirmatory run"}58 continue59 best_ctl_name, best_ctl = max(((k, v) for k, v in cc.items() if is_control(k)), key=lambda kv: kv[1]["accuracy"])60 pre_name, pre = max(((k, v) for k, v in cc.items() if is_control(k, amended=False)), key=lambda kv: kv[1]["accuracy"])61 shuffled = {k: v["accuracy"] for k, v in cc.items() if "shuffled_query" in k}62 # paired difference: we only have per-condition accuracies and gain_vs_zero CIs; items are identical queries, so63 # the difference of gains has the same per-item structure; recompute from the stored gain estimates' bootstrap is not64 # possible without per-item vectors -> use the conservative difference of accuracies with a bootstrap on the65 # per-item correctness reconstructed from outputs when available.66 diff = cc[sel]["accuracy"] - best_ctl["accuracy"]67 if "correct" in cc[sel] and "correct" in best_ctl: # exact paired item bootstrap68 sys.path.insert(0, str(ROOT / "src"))69 from wp_ai_0001.statistics.bootstrap import cluster_bootstrap_paired_diff7071 ci = cluster_bootstrap_paired_diff(np.array(cc[sel]["correct"], float), np.array(best_ctl["correct"], float), [str(i) for i in range(len(cc[sel]["correct"]))], 2000)72 lo, hi = ci.low, ci.high73 else: # conservative normal combination of the two paired gain-vs-zero intervals (same items)74 def se(ci):75 return (ci["high"] - ci["low"]) / (2 * 1.96)76 s = (se(cc[sel]["gain_vs_zero"]) ** 2 + se(best_ctl["gain_vs_zero"]) ** 2) ** 0.577 lo, hi = diff - 1.96 * s, diff + 1.96 * s78 ok = elig and lo > 0.1079 n_ok += int(ok)80 rows[tn] = {"eligible": elig, "selected": sel, "dev_acc": selv["accuracy"], "conf_zero": trc["recipient_zero"], "conf_few": trc["recipient_few"], "conf_own": trc["recipient_own_vector"],81 "conf_selected": cc[sel]["accuracy"], "best_control": best_ctl_name, "best_control_acc": best_ctl["accuracy"], "diff": diff, "ci": [lo, hi], "supported": ok,82 "pre_amendment_best_control": pre_name, "pre_amendment_best_control_acc": pre["accuracy"], "pre_amendment_diff": cc[sel]["accuracy"] - pre["accuracy"], "shuffled_query_diagnostic": shuffled}83 L.append(f"| {p} | {tn} | {'yes' if elig else 'no'} ({trd['recipient_own_vector']:.2f}) | {sel} ({selv['accuracy']:.2f}) | {trc['recipient_zero']:.2f} / {trc['recipient_few']:.2f} / {trc['recipient_own_vector']:.2f} / **{cc[sel]['accuracy']:.2f}** / {best_ctl['accuracy']:.2f} ({best_ctl_name}) | {diff:+.2f} [{lo:+.2f}, {hi:+.2f}]{' ✔' if ok else ''}; pre-A1 control {pre['accuracy']:.2f}; shuffled-query diag. {max(shuffled.values()) if shuffled else float('nan'):.2f} |")84 verdict = "supported" if n_ok >= 2 else ("partial" if n_ok == 1 else "not supported")85 out["pairs"][p] = {"tasks": rows, "n_supported_tasks": n_ok, "H6": verdict, "collateral": pr.get("collateral")}86 L.append(f"| **{p}** | | | | | **H6 {verdict} ({n_ok} task(s))** |")87 sdir = ROOT / "experiments" / "summaries" / a.conf88 (sdir / "tier1_confirmatory.json").write_text(json.dumps(out, indent=1, default=float))89 exact = all("correct" in v for pr in conf["pairs"].values() for tr in pr["tasks"].values() for v in tr["conditions"].values())90 L += ["", "Note: " + ("CIs are exact paired item bootstraps (2 000 draws) on per-item correctness stored in the confirmatory results." if exact else "the CI of the selected-minus-control difference uses a conservative normal combination of the two paired gain-vs-zero bootstrap intervals (same items) because per-item correctness is not stored in this run.")]91 out["exact_paired_bootstrap"] = exact92 (sdir / a.out).write_text("\n".join(L) + "\n")93 print("\n".join(L))949596if __name__ == "__main__":97 main()98