WP46: collect-and-evaluate chain
1 changed file +23 −0
added
scripts/collect_and_evaluate.sh
+23 −0
@@ -0,0 +1,23 @@ | |||||
| 1 | +#!/bin/bash | ||||
| 2 | +# Collect the main run from the nodes, compute losses, run the analyses and build the exhibits (resumable; logs in results/logs). | ||||
| 3 | +set -u | ||||
| 4 | +cd "$(dirname "$0")/.." || exit 1 | ||||
| 5 | +PY=.venv/bin/python | ||||
| 6 | +NODES=${NODES:-M3U96a:24,M3U96b:26,M2U64:22,M5M36:16,M4M64a:14,M4M64c:14} | ||||
| 7 | +$PY scripts/dispatch_nodes.py collect --name main --nodes "$NODES" 2>&1 | tee results/logs/collect_main.log | ||||
| 8 | +$PY - <<'PYEOF' | ||||
| 9 | +import json, collections, glob, os | ||||
| 10 | +root="results/forecasts/main" | ||||
| 11 | +files=[f for f in glob.glob(f"{root}/*/*.parquet") if not f.endswith(".fits.parquet")] | ||||
| 12 | +jobs=json.load(open("results/manifests/jobs_main.json")) | ||||
| 13 | +want={(j["symbol"], f"{j['model_id']}__{j['dist']}__w{j['window']}_r{j['refit_every']}") for j in jobs} | ||||
| 14 | +have={(os.path.basename(os.path.dirname(f)), os.path.basename(f)[:-8]) for f in files} | ||||
| 15 | +missing=sorted(want-have) | ||||
| 16 | +print("forecast files", len(have), "of", len(want), "missing", len(missing)) | ||||
| 17 | +json.dump([{"symbol":s,"spec_id":p} for s,p in missing], open("results/manifests/jobs_main_missing.json","w"), indent=0) | ||||
| 18 | +print(collections.Counter(p.split("__")[0] for s,p in missing).most_common(10)) | ||||
| 19 | +PYEOF | ||||
| 20 | +$PY scripts/compute_losses.py --run main --workers 6 2>&1 | tee results/logs/compute_losses_main.log | tail -3 | ||||
| 21 | +$PY scripts/analyze.py --run main --period test 2>&1 | tee results/logs/analyze_main_test.log | tail -5 | ||||
| 22 | +$PY scripts/analyze.py --run main --period validation --reps 500 2>&1 | tee results/logs/analyze_main_validation.log | tail -3 | ||||
| 23 | +$PY scripts/build_paper.py --run main --period test 2>&1 | tee results/logs/build_paper.log | tail -3 | ||||
| 24 | |||||