spb/wpai0002_uqo

1#!/usr/bin/env bash2# The single confirmatory scoring pass (pre-registration v1.0 + A1). Gathers predictions from the nodes, then scores3# every confirmatory set once with 10 000 document-clustered bootstrap resamples and Holm correction.4# Usage: scripts/run_confirmatory.sh [gather]5set -euo pipefail6cd "$(dirname "$0")/.."7PY=.venv/bin/python8if [ "${1:-}" = gather ]; then9  for n in M4M64c M3U96b M4M64a; do10    for d in medium_test medium_heldout_domain medium_heldout_period; do11      rsync -az --ignore-existing "$n:~/wpai0002/artifacts/predictions/$d/" "artifacts/predictions/$d/" 2>/dev/null || true12    done13  done14fi15echo "== systems per set"16for d in medium_test medium_heldout_domain medium_heldout_period heldout_opus heldout_heldout_models heldout_heldout_prompts; do17  echo "$d: $(ls artifacts/predictions/$d 2>/dev/null | grep outputs | sed 's/outputs_//; s/.jsonl//' | tr '\n' ' ')"18done19MED=data/pairs/medium_v1_haiku/validated/pairs.jsonl20$PY scripts/evaluate.py --pairs $MED --outputs-dir artifacts/predictions/medium_test --name conf_test --n-boot 10000 --confirmatory21$PY scripts/evaluate.py --pairs $MED --outputs-dir artifacts/predictions/medium_heldout_domain --name conf_heldout_domain --n-boot 10000 --confirmatory22$PY scripts/evaluate.py --pairs $MED --outputs-dir artifacts/predictions/medium_heldout_period --name conf_heldout_period --n-boot 10000 --confirmatory23$PY scripts/evaluate.py --pairs data/pairs/pilot_opus/validated/pairs.jsonl --outputs-dir artifacts/predictions/heldout_opus --name conf_heldout_opus --n-boot 10000 --confirmatory24$PY scripts/evaluate.py --pairs data/pairs/pilot_heldout_models/validated/pairs.jsonl --outputs-dir artifacts/predictions/heldout_heldout_models --name conf_heldout_models --n-boot 10000 --confirmatory25$PY scripts/evaluate.py --pairs data/pairs/pilot_heldout_prompts/validated/pairs.jsonl --outputs-dir artifacts/predictions/heldout_heldout_prompts --name conf_heldout_prompts --n-boot 10000 --confirmatory26echo "CONFIRMATORY SCORING DONE"27