spb/forge Public MIT
Forge — LLM training from scratch in pure C++20 + Metal on Apple Silicon.
C++ 61.2%
C 23%
Python 7.6%
TeX 7.2%
CMake 1.1%
1{2 "_comment": "MoE variant: 4 experts (d_ff 1728 each) + top-2 softmax router per block, ~152M total params but only ~2 experts' worth active per token. v1 computes all experts densely (correctness first), so steps cost ~4x the dense MLP; the aux load-balance loss (E * sum mean_gate^2, weight 0.01) keeps the router from collapsing. Same data/steps as gpt-50m.",3 "model": {4 "name": "gpt-50m-moe",5 "n_layers": 10,6 "d_model": 640,7 "n_heads": 10,8 "n_kv_heads": 10,9 "d_ff": 1728,10 "vocab_size": 4096,11 "context_length": 1024,12 "tied_embeddings": true,13 "use_rope": true,14 "rope_theta": 10000.0,15 "norm": "rmsnorm",16 "norm_eps": 1e-06,17 "activation": "swiglu",18 "dropout": 0.0,19 "n_experts": 4,20 "moe_top_k": 2,21 "moe_aux_weight": 0.0122 },23 "train": {24 "lr": 0.0005,25 "min_lr_ratio": 0.1,26 "warmup_steps": 117,27 "max_steps": 1170,28 "beta1": 0.9,29 "beta2": 0.95,30 "eps": 1e-08,31 "weight_decay": 0.1,32 "grad_clip": 1.0,33 "batch_size": 8,34 "grad_accum_steps": 8,35 "precision": "f32",36 "checkpoint_every": 200,37 "eval_every": 100,38 "eval_batches": 20,39 "seed": 133740 }41}42