SPB Git
spboucher.ai

Repository Public Research preview

spbwp41_uqo

Commit

P11: per-node compute ledger tables (compute_table.csv, compute_runs.csv) from the gathered ledger and training summaries (WP41-DATA-06)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed · Nov 6, 2026 parent 9060c36
3 changed files +166 −0 Browse files
3 changed files
Unified Split
added results/metrics/compute_runs.csv +45 −0
@@ -0,0 +1,45 @@
1 +exp,model,host,device,steps,max_steps,wall_s,ex_per_s,best_val,val_subset,n_params,peak_mem_mb,seed
2 +ablations,mrpt_small_1m_5m,,mps,4000,,719.5306529998779,355.7874830386739,-6.451227885057247,,3330819,87.04,
3 +ablations,mrpt_small_1m_only_b4k,,mps,4000,,617.6293749809265,414.48805767683206,-6.4504765663426005,,3200259,86.535168,
4 +ablations,mrpt_small_b4k,,mps,4000,,842.46826004982,303.86901458443225,-6.450709887063427,,3455619,62.975488,
5 +ablations,mrpt_small_copula,,mps,4000,,855.1630790233612,299.3581064004362,-6.450911094056047,,3456584,91.215104,
6 +ablations,mrpt_small_cross,,mps,4000,,793.9436521530151,322.4410187118187,-6.45034669881108,,5533443,122.998784,
7 +ablations,mrpt_small_gaussian,,mps,4000,,852.7440068721771,300.20732826841595,-6.3999923274301365,,3455619,107.833344,
8 +ablations,mrpt_small_learnedregime,,mps,4000,,853.3193740844727,300.004907628709,-6.450148327536085,,3421643,90.356224,
9 +ablations,mrpt_small_noregime,,mps,4000,,854.9879620075226,299.4194203610876,-6.450862349156063,,3416643,90.587136,
10 +ablations,mrpt_small_ohlcv,,mps,4000,,855.4139831066132,299.2703007616066,-6.449995007517438,,3455619,91.210752,
11 +ablations,mrpt_small_returns_only,,mps,4000,,854.0778880119324,299.73847068667277,-6.446453059924354,,3455619,90.939392,
12 +ablations,mrpt_small_seed2,,mps,4000,,857.5486867427826,298.52532451814704,-6.451273006874564,,3455619,91.579392,
13 +ablations,mrpt_small_seed3,,mps,4000,,854.6393349170685,299.54156044647937,-6.450616433821653,,3455619,91.210752,
14 +ablations,mrpt_small_shuffled,,mps,4000,,853.1240651607513,300.07358888857834,-6.449357611608497,,3455619,91.210752,
15 +ablations,mrpt_small_ticker,,mps,4000,,853.8766720294952,299.8091040378687,-6.451585810243266,,3501699,91.948032,
16 +ablations,mrpt_small_time_only,,mps,4000,,853.4852628707886,299.9465967800273,-6.44639632448096,,3455619,91.210752,
17 +ablations_h3,mrpt_small_redglob_none,MacStudnpierre2,mps,4000,4000.0,639.7647252082825,400.1471008215659,-6.118772100431643,50000.0,3416643,62.155264,1.0
18 +ablations_h3,mrpt_small_redglob_stats,MacStudnpierre2,mps,4000,4000.0,640.1490380764008,399.9068728889456,-6.118654915594382,50000.0,3455619,62.975488,1.0
19 +ablations_h3,mrpt_small_shuffled_pv,M5M36,mps,4000,4000.0,932.8898439407349,274.4161078210466,-6.117766850176689,50000.0,3455619,62.975488,1.0
20 +ablations_h3,mrpt_small_time_only_redglob,M5M36,mps,4000,4000.0,936.1511559486389,273.46011204844916,-6.10712526195645,50000.0,3455619,62.909952,1.0
21 +budget,mrpt_small_1m_only_c24k,M5M36,mps,20000,24000.0,743.2170901298523,688.8969680588818,-6.119870707857929,50000.0,3200259,71.195392,1.0
22 +budget,mrpt_small_1m_only_c36k,M5M36,mps,30000,36000.0,912.2271790504456,701.5796225959723,-6.120694511013528,50000.0,3200259,71.195392,1.0
23 +budget,mrpt_small_c24k,MacStudnpierre2,mps,24000,24000.0,1927.6599280834198,398.41052294093475,-6.122068936847249,50000.0,3455619,77.02912,1.0
24 +budget,mrpt_small_c36k,MacStudnpierre2,mps,34000,36000.0,1563.6283888816833,409.30441308866995,-6.122311429211835,50000.0,3455619,77.02912,1.0
25 +main,mlp,,mps,12000,,838.5925340652466,915.8202211471702,-6.449690639457265,,301827,14.961152,
26 +main,mrpt_base,,mps,12000,,4579.964827299118,167.6868772926587,-6.452572455371413,,20718339,402.962432,
27 +main,mrpt_small,,mps,12000,,1759.3876690864563,436.5155067835482,-6.453461130264837,,3455619,63.172096,
28 +main,mrpt_small_1m_only,,mps,12000,,1087.6674129962921,706.0981976874011,-6.452020036474576,,3200259,87.387136,
29 +main,tcn,,mps,12000,,927.3784019947052,828.1409167477947,-6.451788332870508,,231459,199.197696,
30 +pilot,mlp,,mps,600,,151.7184603214264,253.10038026122106,-6.066584629400399,,301827,14.961152,
31 +pilot,mrpt_small,,mps,600,,206.8860673904419,185.6094056229041,-6.066347980807233,,3455619,62.975488,
32 +pilot,mrpt_small_1m_only,,mps,600,,168.56347799301147,227.80735457767452,-6.065922264944626,,3200259,86.469632,
33 +pilot,tcn,,mps,600,,157.81467294692993,243.32338231257614,-6.068539780236767,,231459,35.361792,
34 +seeds,mrpt_small_1m_only_s2,M4M64c,mps,12000,12000.0,931.623507976532,824.3673473505204,-6.119955929171179,50000.0,3200259,58.36544,2.0
35 +seeds,mrpt_small_1m_only_s3,M4M64c,mps,12000,12000.0,931.3172252178192,824.6384574497458,-6.119932442382342,50000.0,3200259,58.69312,3.0
36 +seeds,mrpt_small_1m_only_s4,M4M64c,mps,12000,12000.0,938.0887632369995,818.6858537244539,-6.119824877009311,50000.0,3200259,58.430976,4.0
37 +seeds,mrpt_small_1m_only_s5,M4M64c,mps,12000,12000.0,932.3040630817413,823.7655829380037,-6.119784669494894,50000.0,3200259,58.791424,5.0
38 +seeds,mrpt_small_s2,MacStudnpierre6,mps,12000,12000.0,2062.317788362503,372.39653574912825,-6.121164594482364,50000.0,3455619,62.647808,2.0
39 +seeds,mrpt_small_s3,MacStudnpierre6,mps,12000,12000.0,2023.1792509555817,379.60057154464226,-6.120792555880995,50000.0,3455619,63.172096,3.0
40 +seeds,mrpt_small_s4,MacStudnpierre4,mps,12000,12000.0,2033.7075672149658,377.6354144424646,-6.120700168242146,50000.0,3455619,62.909952,4.0
41 +seeds,mrpt_small_s5,MacStudnpierre4,mps,12000,12000.0,2023.9454181194305,379.45687325579905,-6.121044530367436,50000.0,3455619,62.647808,5.0
42 +synthetic_smoke,mlp,,mps,60,,23.60936999320984,81.32364398339305,-6.087464488115836,,301827,11.138048,
43 +synthetic_smoke,mrpt_tiny,,mps,60,,24.997076988220215,76.80898054219672,-6.0867066191312835,,190147,5.631488,
44 +synthetic_smoke,mrpt_tiny_copula,,mps,60,,24.7404522895813,77.60569522039621,-6.0867741598669935,,190342,7.154432,
45 +transfer300,mrpt_small_all300,m3u96b,mps,12000,12000.0,1901.9389271736145,403.7984548438104,-6.12384876618515,50000.0,3455619,62.647808,1.0
added results/metrics/compute_table.csv +84 −0
@@ -0,0 +1,84 @@
1 +job,node,chip,start,end,wall_s,exit,code_sha256_16
2 +P3_mrpt_small_s2,MacStudnpierre6,Apple M4 Max,2026-11-06T04:55:45Z,2026-11-06T05:41:10Z,2725,0,36b17d2853e248f2
3 +P3_mrpt_small_1m_only_s2,M4M64c,Apple M4 Max,2026-11-06T04:55:45Z,2026-11-06T05:17:59Z,1334,0,36b17d2853e248f2
4 +P2_copula_only,m3u96b,Apple M3 Ultra,2026-11-06T04:57:27Z,2026-11-06T06:11:28Z,4441,0,d71b299a519f6567
5 +P1_pairwise_ablations_test_pass1,m4mm,Apple M4 Pro,2026-11-06T05:00:30Z,2026-11-06T05:04:02Z,212,0,ada08721e76115a7
6 +P1_pairwise_main_test,MacStudonpierre,Apple M2 Ultra,2026-11-06T05:00:58Z,2026-11-06T05:05:24Z,266,0,3185a6b7b341f089
7 +P5_baselines_main_test,m4mn,Apple M4 Pro,2026-11-06T05:01:00Z,2026-11-06T05:28:42Z,1661,0,3185a6b7b341f089
8 +P4_mrpt_small_1m_only_c24k,M5M36,Apple M5 Max,2026-11-06T05:01:21Z,2026-11-06T05:20:00Z,1119,0,3185a6b7b341f089
9 +P8_download66,m3u96b,Apple M3 Ultra,2026-11-06T05:03:54Z,2026-11-06T05:06:04Z,130,0,3185a6b7b341f089
10 +P1_tables_ablations_test_pass1,m4mm,Apple M4 Pro,2026-11-06T05:04:02Z,2026-11-06T05:06:40Z,158,0,ada08721e76115a7
11 +P1_tables_main_test,MacStudonpierre,Apple M2 Ultra,2026-11-06T05:05:24Z,2026-11-06T05:08:00Z,156,0,3185a6b7b341f089
12 +P4_mrpt_small_c24k,MacStudnpierre2,Apple M3 Ultra,2026-11-06T05:05:30Z,2026-11-06T05:47:16Z,2506,0,3185a6b7b341f089
13 +P3_mrpt_small_s4,MacStudnpierre4,Apple M4 Max,2026-11-06T05:05:40Z,2026-11-06T05:50:30Z,2690,0,3185a6b7b341f089
14 +P8_baselines_test,m3u96b,Apple M3 Ultra,2026-11-06T05:06:04Z,2026-11-06T05:08:44Z,160,0,3185a6b7b341f089
15 +P7_copula_diagnostics,m4mm,Apple M4 Pro,2026-11-06T05:06:40Z,2026-11-06T05:07:08Z,28,0,ada08721e76115a7
16 +P1_pairwise_main_test_heldout,MacStudonpierre,Apple M2 Ultra,2026-11-06T05:08:00Z,2026-11-06T05:09:11Z,71,0,3185a6b7b341f089
17 +P8_baselines_final,m3u96b,Apple M3 Ultra,2026-11-06T05:08:44Z,2026-11-06T05:12:01Z,197,0,3185a6b7b341f089
18 +P1_tables_main_test_heldout,MacStudonpierre,Apple M2 Ultra,2026-11-06T05:09:11Z,2026-11-06T05:09:42Z,31,0,3185a6b7b341f089
19 +P1_pairwise_final_final,MacStudonpierre,Apple M2 Ultra,2026-11-06T05:09:42Z,2026-11-06T05:14:11Z,269,0,3185a6b7b341f089
20 +P8_coverage_2017H2_2022,m3u96b,Apple M3 Ultra,2026-11-06T05:12:01Z,2026-11-06T05:18:02Z,361,0,3185a6b7b341f089
21 +P1_tables_final_final,MacStudonpierre,Apple M2 Ultra,2026-11-06T05:14:11Z,2026-11-06T05:16:45Z,154,0,3185a6b7b341f089
22 +P1_pairwise_final_final_heldout,MacStudonpierre,Apple M2 Ultra,2026-11-06T05:16:45Z,2026-11-06T05:17:52Z,67,0,3185a6b7b341f089
23 +P1_tables_final_final_heldout,MacStudonpierre,Apple M2 Ultra,2026-11-06T05:17:52Z,2026-11-06T05:18:26Z,34,0,3185a6b7b341f089
24 +P3_mrpt_small_1m_only_s3,M4M64c,Apple M4 Max,2026-11-06T05:17:59Z,2026-11-06T05:40:08Z,1329,0,36b17d2853e248f2
25 +P8_instrument_status,m3u96b,Apple M3 Ultra,2026-11-06T05:18:02Z,2026-11-06T05:19:38Z,96,1,3185a6b7b341f089
26 +P1_mde,MacStudonpierre,Apple M2 Ultra,2026-11-06T05:18:26Z,2026-11-06T05:18:26Z,0,0,3185a6b7b341f089
27 +P1_convention,MacStudonpierre,Apple M2 Ultra,2026-11-06T05:18:26Z,2026-11-06T05:18:27Z,1,0,3185a6b7b341f089
28 +P4_mrpt_small_1m_only_c36k,M5M36,Apple M5 Max,2026-11-06T05:20:00Z,2026-11-06T05:41:26Z,1285,0,3185a6b7b341f089
29 +P8_instrument_status,m3u96b,Apple M3 Ultra,2026-11-06T05:22:17Z,2026-11-06T05:23:29Z,72,0,d71b299a519f6567
30 +P8_origin_eligibility,m3u96b,Apple M3 Ultra,2026-11-06T05:23:29Z,2026-11-06T05:23:30Z,1,1,d71b299a519f6567
31 +P8_origin_eligibility,m3u96b,Apple M3 Ultra,2026-11-06T05:25:09Z,2026-11-06T05:25:28Z,19,0,d71b299a519f6567
32 +P5_baselines_main_test_heldout,m4mn,Apple M4 Pro,2026-11-06T05:28:42Z,2026-11-06T05:36:01Z,439,0,3185a6b7b341f089
33 +P5_baselines_final_final,m4mn,Apple M4 Pro,2026-11-06T05:36:01Z,2026-11-06T06:04:07Z,1686,0,3185a6b7b341f089
34 +P3_mrpt_small_1m_only_s4,M4M64c,Apple M4 Max,2026-11-06T05:40:08Z,2026-11-06T06:02:24Z,1336,0,36b17d2853e248f2
35 +P3_mrpt_small_s3,MacStudnpierre6,Apple M4 Max,2026-11-06T05:41:10Z,2026-11-06T06:25:40Z,2670,0,36b17d2853e248f2
36 +P10_time_only_redglob,M5M36,Apple M5 Max,2026-11-06T05:41:26Z,2026-11-06T06:07:53Z,1587,0,3185a6b7b341f089
37 +P4_mrpt_small_c36k,MacStudnpierre2,Apple M3 Ultra,2026-11-06T05:47:16Z,2026-11-06T06:22:45Z,2129,0,3185a6b7b341f089
38 +P3_mrpt_small_s5,MacStudnpierre4,Apple M4 Max,2026-11-06T05:50:30Z,2026-11-06T06:35:02Z,2672,0,3185a6b7b341f089
39 +P3_mrpt_small_1m_only_s5,M4M64c,Apple M4 Max,2026-11-06T06:02:24Z,2026-11-06T06:24:35Z,1331,0,36b17d2853e248f2
40 +P5_baselines_final_final_heldout,m4mn,Apple M4 Pro,2026-11-06T06:04:07Z,2026-11-06T06:11:32Z,445,0,3185a6b7b341f089
41 +P10_shuffled_pv,M5M36,Apple M5 Max,2026-11-06T06:07:53Z,2026-11-06T06:34:15Z,1582,0,3185a6b7b341f089
42 +P9_transfer300_train,m3u96b,Apple M3 Ultra,2026-11-06T06:11:28Z,2026-11-06T06:57:16Z,2748,0,d71b299a519f6567
43 +P5_recal_subset,m4mn,Apple M4 Pro,2026-11-06T06:11:32Z,2026-11-06T06:12:01Z,29,0,3185a6b7b341f089
44 +P5_recal_parametric,m4mn,Apple M4 Pro,2026-11-06T06:12:01Z,2026-11-06T06:13:58Z,117,0,3185a6b7b341f089
45 +P5_recal_floor,m4mn,Apple M4 Pro,2026-11-06T06:13:58Z,2026-11-06T06:13:59Z,1,0,3185a6b7b341f089
46 +P5_rescore_parametric,m4mn,Apple M4 Pro,2026-11-06T06:13:59Z,2026-11-06T06:50:42Z,2203,0,3185a6b7b341f089
47 +P8_neural_test,MacStudonpierre,Apple M2 Ultra,2026-11-06T06:20:29Z,2026-11-06T06:28:56Z,507,0,d71b299a519f6567
48 +P10_redglob_stats,MacStudnpierre2,Apple M3 Ultra,2026-11-06T06:22:46Z,2026-11-06T06:42:54Z,1208,0,3185a6b7b341f089
49 +P8_tables_test,MacStudonpierre,Apple M2 Ultra,2026-11-06T06:28:56Z,2026-11-06T06:29:04Z,8,0,d71b299a519f6567
50 +P8_pairwise_test,MacStudonpierre,Apple M2 Ultra,2026-11-06T06:29:04Z,2026-11-06T06:29:14Z,10,0,d71b299a519f6567
51 +P8_neural_final,MacStudonpierre,Apple M2 Ultra,2026-11-06T06:29:14Z,2026-11-06T06:38:58Z,584,0,d71b299a519f6567
52 +P8_tables_final,MacStudonpierre,Apple M2 Ultra,2026-11-06T06:38:58Z,2026-11-06T06:39:08Z,10,0,d71b299a519f6567
53 +P8_pairwise_final,MacStudonpierre,Apple M2 Ultra,2026-11-06T06:39:08Z,2026-11-06T06:39:20Z,12,0,d71b299a519f6567
54 +P5_recal_subset,MacStudonpierre,Apple M2 Ultra,2026-11-06T06:39:20Z,2026-11-06T06:40:02Z,42,0,d71b299a519f6567
55 +P5_recal_neural_predict,MacStudonpierre,Apple M2 Ultra,2026-11-06T06:40:02Z,2026-11-06T06:49:21Z,559,0,d71b299a519f6567
56 +P3_seed_contrast,m3u96b,Apple M3 Ultra,2026-11-06T06:40:34Z,2026-11-06T06:40:50Z,16,0,d71b299a519f6567
57 +P3_pairwise_seeds,m3u96b,Apple M3 Ultra,2026-11-06T06:40:50Z,2026-11-06T06:42:55Z,125,0,d71b299a519f6567
58 +P10_redglob_none,MacStudnpierre2,Apple M3 Ultra,2026-11-06T06:42:54Z,2026-11-06T07:03:02Z,1208,0,3185a6b7b341f089
59 +P3_tables_seeds,m3u96b,Apple M3 Ultra,2026-11-06T06:42:55Z,2026-11-06T06:43:03Z,8,0,d71b299a519f6567
60 +P5_recal_neural_fit,MacStudonpierre,Apple M2 Ultra,2026-11-06T06:49:21Z,2026-11-06T06:54:54Z,333,0,d71b299a519f6567
61 +P5_legacy_check,m4mn,Apple M4 Pro,2026-11-06T06:50:42Z,2026-11-06T07:18:34Z,1672,0,3185a6b7b341f089
62 +P9_transfer300_heldout_test,m3u96b,Apple M3 Ultra,2026-11-06T06:57:16Z,2026-11-06T07:00:28Z,192,0,d71b299a519f6567
63 +P9_transfer300_heldout_final,m3u96b,Apple M3 Ultra,2026-11-06T07:00:28Z,2026-11-06T07:03:42Z,194,0,d71b299a519f6567
64 +P5_recal_subset,m3u96b,Apple M3 Ultra,2026-11-06T07:03:42Z,2026-11-06T07:04:11Z,29,0,d71b299a519f6567
65 +P5_recal_neural_predict,m3u96b,Apple M3 Ultra,2026-11-06T07:04:11Z,2026-11-06T07:04:31Z,20,0,d71b299a519f6567
66 +P5_recal_neural_fit,m3u96b,Apple M3 Ultra,2026-11-06T07:04:31Z,2026-11-06T07:09:18Z,286,0,0da9f84f4ba8667d
67 +P4_tables_budget,m3u96b,Apple M3 Ultra,2026-11-06T07:04:51Z,2026-11-06T07:04:55Z,4,0,0da9f84f4ba8667d
68 +P4_learning_curve,m3u96b,Apple M3 Ultra,2026-11-06T07:04:55Z,2026-11-06T07:05:01Z,6,0,0da9f84f4ba8667d
69 +P1_pairwise_ablations_test_pass2,m3u96b,Apple M3 Ultra,2026-11-06T07:05:01Z,2026-11-06T07:09:06Z,245,0,0da9f84f4ba8667d
70 +P1_tables_ablations_test_pass2,m3u96b,Apple M3 Ultra,2026-11-06T07:09:06Z,2026-11-06T07:11:53Z,167,0,0da9f84f4ba8667d
71 +P5_recal_merge,m3u96b,Apple M3 Ultra,2026-11-06T07:29:42Z,2026-11-06T07:29:43Z,1,0,0da9f84f4ba8667d
72 +P10_pairwise,m3u96b,Apple M3 Ultra,2026-11-06T07:29:59Z,2026-11-06T07:30:29Z,30,1,0da9f84f4ba8667d
73 +P10_pairwise,m3u96b,Apple M3 Ultra,2026-11-06T07:33:16Z,2026-11-06T07:35:03Z,107,0,0da9f84f4ba8667d
74 +P10_tables,m3u96b,Apple M3 Ultra,2026-11-06T07:35:03Z,2026-11-06T07:35:07Z,4,0,0da9f84f4ba8667d
75 +P1_pairwise_main_test_final,m3u96b,Apple M3 Ultra,2026-11-06T07:35:07Z,2026-11-06T07:38:43Z,216,0,0da9f84f4ba8667d
76 +P1_tables_main_test_final,m3u96b,Apple M3 Ultra,2026-11-06T07:38:43Z,2026-11-06T07:41:54Z,191,0,0da9f84f4ba8667d
77 +P1_pairwise_main_test_heldout_final,m3u96b,Apple M3 Ultra,2026-11-06T07:41:54Z,2026-11-06T07:42:57Z,63,0,0da9f84f4ba8667d
78 +P1_tables_main_test_heldout_final,m3u96b,Apple M3 Ultra,2026-11-06T07:42:57Z,2026-11-06T07:43:33Z,35,0,0da9f84f4ba8667d
79 +P1_pairwise_final_final_final,m3u96b,Apple M3 Ultra,2026-11-06T07:43:33Z,2026-11-06T07:47:15Z,222,0,0da9f84f4ba8667d
80 +P1_tables_final_final_final,m3u96b,Apple M3 Ultra,2026-11-06T07:47:15Z,2026-11-06T07:50:12Z,177,0,0da9f84f4ba8667d
81 +P1_pairwise_final_final_heldout_final,m3u96b,Apple M3 Ultra,2026-11-06T07:50:12Z,2026-11-06T07:51:16Z,64,0,0da9f84f4ba8667d
82 +P1_tables_final_final_heldout_final,m3u96b,Apple M3 Ultra,2026-11-06T07:51:16Z,2026-11-06T07:51:52Z,36,0,0da9f84f4ba8667d
83 +P1_mde_final,m3u96b,Apple M3 Ultra,2026-11-06T07:51:52Z,2026-11-06T07:51:52Z,0,0,0da9f84f4ba8667d
84 +P1_convention_final,m3u96b,Apple M3 Ultra,2026-11-06T07:51:52Z,2026-11-06T07:51:53Z,1,0,0da9f84f4ba8667d
added scripts/review/compute_table.py +37 −0
@@ -0,0 +1,37 @@
1 +"""Review P11: per-node compute ledger -> results/metrics/compute_table.csv (job, node, chip, wall_s, exit) and
2 +results/metrics/compute_runs.csv (every new training run: exp, model, host, device, steps, wall_s, ex_per_s, best_val,
3 +val_subset). Reads results/metrics/review_runs*.jsonl and results/metrics/*/*/train_summary.json."""
4 +from __future__ import annotations
5 +
6 +import json
7 +from pathlib import Path
8 +
9 +import pandas as pd
10 +
11 +ROOT = Path(__file__).resolve().parents[2]
12 +
13 +
14 +def main() -> None:
15 + rows = []
16 + for f in (ROOT / "results/metrics/review_runs.jsonl", ROOT / "results/metrics/review_runs_gathered.jsonl"):
17 + if f.exists():
18 + rows += [json.loads(l) for l in f.read_text().splitlines() if l.strip()]
19 + led = pd.DataFrame(rows).drop_duplicates(subset=["node", "job", "start"])
20 + led = led.sort_values(["start"])
21 + led[["job", "node", "chip", "start", "end", "wall_s", "exit", "code_sha256_16"]].to_csv(ROOT / "results/metrics/compute_table.csv", index=False)
22 + runs = []
23 + for f in sorted((ROOT / "results/metrics").glob("*/*/train_summary.json")):
24 + js = json.loads(f.read_text())
25 + runs.append({"exp": f.parent.parent.name, "model": f.parent.name, "host": js.get("host"), "device": js.get("device"), "steps": js.get("steps"), "max_steps": js.get("max_steps"),
26 + "wall_s": js.get("wall_s"), "ex_per_s": js.get("ex_per_s"), "best_val": js.get("best_val"), "val_subset": js.get("val_subset"), "n_params": js.get("n_params"),
27 + "peak_mem_mb": js.get("peak_mem_mb"), "seed": js.get("seed")})
28 + r = pd.DataFrame(runs); r.to_csv(ROOT / "results/metrics/compute_runs.csv", index=False)
29 + by_node = led.groupby(["node", "chip"]).agg(jobs=("job", "count"), wall_h=("wall_s", lambda x: round(x.sum() / 3600, 2)), failures=("exit", lambda x: int((x != 0).sum()))).reset_index()
30 + print(by_node.to_string(index=False)); print(f"total ledger wall {led.wall_s.sum() / 3600:.2f} h over {len(led)} jobs")
31 + new = r[r.host.notna()]
32 + print(new[["exp", "model", "host", "steps", "wall_s", "ex_per_s", "best_val"]].round(3).to_string(index=False))
33 + print(f"new training runs: {len(new)}, GPU train wall {new.wall_s.sum() / 3600:.2f} h")
34 +
35 +
36 +if __name__ == "__main__":
37 + main()
38
Menu

Theme
↑↓navigate ↵open escclose /⌘Kanywhere