spb/forge Public MIT
Forge — LLM training from scratch in pure C++20 + Metal on Apple Silicon.
C++ 61.2%
C 23%
Python 7.6%
TeX 7.2%
CMake 1.1%
1{2 "_comment": "gpt-50m with BitNet-style ternary QAT: every linear weight is fake-quantized to {-s, 0, +s} (per-row absmean scale) each forward, gradients flow via straight-through estimator into f32 master weights. Embeddings/lm_head stay f32. Same data/steps as gpt-50m for A/B comparison.",3 "model": {4 "name": "gpt-50m-ternary",5 "n_layers": 10,6 "d_model": 640,7 "n_heads": 10,8 "n_kv_heads": 10,9 "d_ff": 1728,10 "vocab_size": 4096,11 "context_length": 1024,12 "tied_embeddings": true,13 "use_rope": true,14 "rope_theta": 10000.0,15 "norm": "rmsnorm",16 "norm_eps": 1e-06,17 "activation": "swiglu",18 "dropout": 0.0,19 "quant": "ternary"20 },21 "train": {22 "lr": 0.0005,23 "min_lr_ratio": 0.1,24 "warmup_steps": 117,25 "max_steps": 1170,26 "beta1": 0.9,27 "beta2": 0.95,28 "eps": 1e-08,29 "weight_decay": 0.1,30 "grad_clip": 1.0,31 "batch_size": 8,32 "grad_accum_steps": 8,33 "precision": "f32",34 "checkpoint_every": 200,35 "eval_every": 100,36 "eval_batches": 20,37 "seed": 133738 }39}40