Add 205.6M-parameter config for the M3 Ultra cluster run

vocab 4096 (matches the proven BPE pipeline; the trainer's pair-count array is
O(V^2) and the encoder O(V*N), which does not scale to 16384 over 400M tokens),
with d_ff 3072 restoring the parameter count to 205.6M. Micro-batch 16 rather
than 4 since 96 GB affords it and it cuts sync points 4x.

Simon-Pierre Boucher committed 10 days ago (Jul 31, 2026) parent af48866

Showing 1 changed file with +38 and −0

added configs/gpt-200m-cluster.json +38 −0

		`@@ -0,0 +1,38 @@`
	1	+{
	2	+ "_comment": "205.6M params for the M3U96a run (Apple M3 Ultra, 60-core GPU, 96 GB, 77.8 GB working set). vocab 4096 keeps the tied embedding cheap and matches the proven BPE pipeline; d_ff 3072 brings the total back over 200M. batch_size is the MICRO-batch: 16 x 16 x 1024 = 262144 tokens/step. Micro-batch 16 (not 4) because 96 GB affords it and it cuts sync points 4x. precision is parsed but not yet honored - all kernels are f32.",
	3	+ "model": {
	4	+ "name": "gpt-200m-cluster",
	5	+ "n_layers": 16,
	6	+ "d_model": 1024,
	7	+ "n_heads": 16,
	8	+ "n_kv_heads": 8,
	9	+ "d_ff": 3072,
	10	+ "vocab_size": 4096,
	11	+ "context_length": 1024,
	12	+ "tied_embeddings": true,
	13	+ "use_rope": true,
	14	+ "rope_theta": 10000.0,
	15	+ "norm": "rmsnorm",
	16	+ "norm_eps": 1e-06,
	17	+ "activation": "swiglu",
	18	+ "dropout": 0.0
	19	+ },
	20	+ "train": {
	21	+ "lr": 3e-4,
	22	+ "min_lr_ratio": 0.1,
	23	+ "warmup_steps": 150,
	24	+ "max_steps": 1500,
	25	+ "beta1": 0.9,
	26	+ "beta2": 0.95,
	27	+ "eps": 1e-08,
	28	+ "weight_decay": 0.1,
	29	+ "grad_clip": 1.0,
	30	+ "batch_size": 16,
	31	+ "grad_accum_steps": 16,
	32	+ "precision": "f32",
	33	+ "checkpoint_every": 50,
	34	+ "eval_every": 50,
	35	+ "eval_batches": 10,
	36	+ "seed": 1337
	37	+ }
	38	+}
	39