spb/wpai0002_uqo

1#!/usr/bin/env bash2# "Full-scale" training chain on a single H100 (bf16 autocast): HTR-Base (335 M) and HTR-Large (1.07 B) denoising3# pretraining run one after the other, each followed by pair fine-tuning and val/test predictions. Run ON the instance.4set -uo pipefail5cd ~/wpai00026export PATH="$HOME/.local/bin:$PATH"7P=data/pairs/medium_v1_haiku/validated/pairs.jsonl; S=data/splits/medium_v1/splits.jsonl; T=artifacts/tokenizers/medium_v08COR=data/parsed/medium_v0; SP0=data/splits/medium_v0/splits.jsonl9run_size() {  # $1 size  $2 denoise steps  $3 denoise max_tokens  $4 ft lr  $5 ft max_tokens10  local SZ=$1 DS=$2 DT=$3 LR=$4 FT=$511  .venv/bin/python scripts/pretrain_denoise.py --corpus $COR --splits $SP0 --tokenizer $T --run denoise_m1_${SZ}_lambda --arch m1 --size $SZ --max-steps $DS --eval-every 1000 --warmup 1500 --lr 2e-4 --variants-per-passage 2 --max-tokens $DT --max-batch 128 --patience 6 --device cuda --amp bf16 --num-workers 16 > train_denoise_${SZ}.log 2>&112  .venv/bin/python scripts/train.py --pairs $P --splits $S --tokenizer $T --run medium_m1_${SZ}_lambda --preset m1 --size $SZ --init-from artifacts/runs/denoise_m1_${SZ}_lambda/best --lr $LR --max-steps 30000 --eval-every 1000 --warmup 500 --max-tokens $FT --max-batch 128 --patience 6 --device cuda --amp bf16 --num-workers 16 > train_medium_${SZ}.log 2>&113  .venv/bin/python scripts/predict.py --pairs $P --splits $S --split val --checkpoint artifacts/runs/medium_m1_${SZ}_lambda/best --system htr-m1-${SZ}-lambda --out artifacts/predictions/medium_val --limit 3000 --device cuda --batch-size 64 > predict_${SZ}_val.log 2>&114  for SPL in test heldout_domain heldout_period; do15    .venv/bin/python scripts/predict.py --pairs $P --splits $S --split $SPL --checkpoint artifacts/runs/medium_m1_${SZ}_lambda/best --system htr-m1-${SZ}-lambda --out artifacts/predictions/medium_${SPL} --device cuda --batch-size 64 > predict_${SZ}_${SPL}.log 2>&116  done17  echo "CHAIN ${SZ} DONE" >> chain_done.txt18}19# Sequential: Base + Large concurrently OOM on one 80 GB H100 (Base alone took 53 GB at 16 384 tokens).20run_size base 30000 16384 1e-4 1228821run_size large 40000 12288 6e-5 819222echo "ALL CHAINS DONE" >> chain_done.txt23