spb/wpai0002_uqo

1#!/usr/bin/env bash2# Predictions on the held-out REWRITER sets (pilot corpus, test split): Opus 5.5 rewrites (pilot_opus), Sonnet/Haiku3# (pilot_heldout_models) and held-out prompts H1/H2 (pilot_heldout_prompts). Scoring happens once, later.4# Usage: scripts/predict_heldout_pilot.sh <system1>=<ckpt1> [...]5set -euo pipefail6cd "$(dirname "$0")/.."7PY=.venv/bin/python8SPLITS=data/splits/pilot_pmc/splits.jsonl9for JOB in pilot_opus pilot_heldout_models pilot_heldout_prompts; do10  PAIRS=data/pairs/$JOB/validated/pairs.jsonl11  OUT=artifacts/predictions/heldout_${JOB#pilot_}12  mkdir -p "$OUT"13  [ -f "$OUT/outputs_B0_identity.jsonl" ] || $PY scripts/predict.py --pairs "$PAIRS" --splits "$SPLITS" --split test --baselines --out "$OUT"14  for SPEC in "$@"; do15    NAME=${SPEC%%=*}; CK=${SPEC#*=}16    [ -f "$OUT/outputs_${NAME}.jsonl" ] && continue17    $PY scripts/predict.py --pairs "$PAIRS" --splits "$SPLITS" --split test --checkpoint "$CK" --system "$NAME" --out "$OUT"18  done19done20echo "HELDOUT PREDICTIONS DONE"21