spb/wp16_uqo

1#!/bin/zsh2# Rétro-évaluation complète répartie sur le cluster : fragments proportionnels aux cœurs, rassemblement sur M2U64.3# Prérequis : 01-03 terminés (modèles trimestriels présents sur M2U64 dans out/$V), env Python sur chaque nœud.4set -e5V=${CV_VERSION:-2026-10-10}6HUB=M2U647typeset -A SHARDS; SHARDS=(M3U96a 4 M3U96b 4 M2U64 3 M4M64a 2 M4M64b 2)8typeset -A CORES;  CORES=(M3U96a 32 M3U96b 32 M2U64 24 M4M64a 16 M4M64b 16)9N=1510PYHUB='~/uqo-eval-terrains/env/bin/python'11echo "== 1. modèles trimestriels → $HUB → nœuds"12rm -rf /tmp/cv-models && mkdir -p /tmp/cv-models13for n in M2U64 M3U96a M3U96b M4M64a M4M64b; do ssh $n "cd ~/comparables-validation/out/$V && tar czf - models_*.json gbm_pred_*.npy gbm_role_pred_*.npy asof.json 2>/dev/null" | tar xzf - -C /tmp/cv-models; done14ls /tmp/cv-models | wc -l15for n in M2U64 M3U96a M3U96b M4M64a M4M64b; do tar czf - -C /tmp/cv-models . | ssh $n "tar xzf - -C ~/comparables-validation/out/$V" & done; wait16for n in M2U64 M3U96a M3U96b M4M64a M4M64b; do echo "  $n: $(ssh $n "ls ~/comparables-validation/out/$V/models_*.json | wc -l") modèles ; rm -f shards"; ssh $n "find ~/comparables-validation/out -maxdepth 1 -name 'shard-*.log' -delete; find ~/comparables-validation/out/$V -name '*shard*' -delete; true"; done17echo "== 2. lancement des $N fragments"18i=019for n in ${(k)SHARDS}; do20  k=$SHARDS[$n]; procs=$(( CORES[$n] / k ))21  py='../env/bin/python'; [ "$n" = "M2U64" ] && py=$PYHUB22  for j in $(seq 1 $k); do23    ssh $n "cd ~/comparables-validation/pipeline && CV_VERSION=$V CV_SHARD=$i/$N CV_PROCS=$procs nohup $py 04_backtest.py > ../out/shard-$i.log 2>&1 &"24    i=$((i+1))25  done26done27echo "== 3. attente"28while true; do29  done_=030  for n in ${(k)SHARDS}; do done_=$(( done_ + $(ssh $n "cd ~/comparables-validation/out && grep -l 'rétro-évaluation terminée' \$(ls | grep shard-) 2>/dev/null | wc -l; true") )); done31  err=0; for n in ${(k)SHARDS}; do err=$(( err + $(ssh $n "cd ~/comparables-validation/out && grep -l Traceback \$(ls | grep shard-) 2>/dev/null | wc -l; true") )); done32  echo "  $(date +%T) fragments terminés : $done_/$N (erreurs : $err)"33  [ "$err" -gt 0 ] && { for n in ${(k)SHARDS}; do ssh $n "cd ~/comparables-validation/out && grep -B2 -A8 Traceback \$(ls | grep shard-) 2>/dev/null; true"; done; exit 1; }34  [ "$done_" -ge "$N" ] && break35  sleep 2036done37echo "== 4. rassemblement sur $HUB"38for n in M3U96a M3U96b M4M64a M4M64b; do ssh -A $HUB "rsync -a -e 'ssh -o StrictHostKeyChecking=accept-new' $(ssh $n 'echo simon-pierreboucher@$(ipconfig getifaddr en0)'):comparables-validation/out/$V/'*shard*' ~/comparables-validation/out/$V/" & done; wait39ssh $HUB "cd ~/comparables-validation/pipeline && CV_VERSION=$V $PYHUB 04b_gather.py && CV_VERSION=$V $PYHUB 05_analyze.py && CV_VERSION=$V $PYHUB 06_report.py" 2>&1 | grep -v Warning40echo "== terminé $(date +%T)"41