spb/wpai0002_uqo

1#!/usr/bin/env bash2# Generate predictions (NOT scores) on the confirmatory splits for a list of checkpoints. Scoring happens once,3# later, with scripts/evaluate.py --confirmatory after M* is fixed by the frozen validation rule.4# Usage: scripts/predict_confirmatory.sh <pairs> <splits> <out_prefix> <system1>=<ckpt1> [<system2>=<ckpt2> ...]5#   splits predicted: test heldout_domain heldout_period ; outputs → artifacts/predictions/<out_prefix>_<split>/6set -euo pipefail7cd "$(dirname "$0")/.."8PY=.venv/bin/python9PAIRS=$1; SPLITS=$2; PREFIX=$3; shift 310for SP in test heldout_domain heldout_period; do11  OUT=artifacts/predictions/${PREFIX}_${SP}12  mkdir -p "$OUT"13  [ -f "$OUT/outputs_B0_identity.jsonl" ] || $PY scripts/predict.py --pairs "$PAIRS" --splits "$SPLITS" --split "$SP" --baselines --out "$OUT"14  for SPEC in "$@"; do15    NAME=${SPEC%%=*}; CK=${SPEC#*=}16    [ -f "$OUT/outputs_${NAME}.jsonl" ] && { echo "skip $NAME $SP (exists)"; continue; }17    $PY scripts/predict.py --pairs "$PAIRS" --splits "$SPLITS" --split "$SP" --checkpoint "$CK" --system "$NAME" --out "$OUT"18  done19done20echo "PREDICTIONS DONE: $PREFIX"21