spb/wpai0002_uqo

1#!/usr/bin/env bash2# Predict + evaluate one or more checkpoints on the confirmatory splits of a pairs file, with baselines.3# Usage: scripts/run_eval_suite.sh <name> <pairs.jsonl> <splits.jsonl> <split> <n_boot> <ckpt1> [ckpt2 ...]4#   e.g. scripts/run_eval_suite.sh pilot_test data/pairs/pilot_opus/validated/pairs.jsonl data/splits/pilot_pmc/splits.jsonl test 10000 \5#            artifacts/runs/pilot_m1_tiny/best artifacts/runs/pilot_m2_tiny/best6set -euo pipefail7cd "$(dirname "$0")/.."8PY=.venv/bin/python9NAME=$1; PAIRS=$2; SPLITS=$3; SPLIT=$4; NBOOT=$5; shift 510OUT=artifacts/predictions/$NAME11mkdir -p "$OUT"12$PY scripts/predict.py --pairs "$PAIRS" --splits "$SPLITS" --split "$SPLIT" --baselines --out "$OUT"13for CK in "$@"; do14  $PY scripts/predict.py --pairs "$PAIRS" --splits "$SPLITS" --split "$SPLIT" --checkpoint "$CK" --out "$OUT"15done16$PY scripts/evaluate.py --pairs "$PAIRS" --outputs-dir "$OUT" --name "$NAME" --n-boot "$NBOOT" --confirmatory17echo "EVAL SUITE DONE: artifacts/metrics/$NAME"18