spb/forge Public MIT

Forge — LLM training from scratch in pure C++20 + Metal on Apple Silicon.

C++ 61.2% C 23% Python 7.6% TeX 7.2% CMake 1.1%

929 B · 39 lines json

Raw Blame History

1{2  "_comment": "Small-context smoke config for end-to-end verification. The unfused attention kernel materializes [B,H,T,T] probabilities, so keep T modest until the fused flash kernel lands.",3  "model": {4    "name": "gpt-smoke",5    "n_layers": 4,6    "d_model": 256,7    "n_heads": 4,8    "n_kv_heads": 2,9    "d_ff": 704,10    "vocab_size": 4096,11    "context_length": 128,12    "tied_embeddings": true,13    "use_rope": true,14    "rope_theta": 10000.0,15    "norm": "rmsnorm",16    "norm_eps": 1e-6,17    "activation": "swiglu",18    "dropout": 0.019  },20  "train": {21    "lr": 1e-3,22    "min_lr_ratio": 0.1,23    "warmup_steps": 100,24    "max_steps": 2000,25    "beta1": 0.9,26    "beta2": 0.95,27    "eps": 1e-8,28    "weight_decay": 0.1,29    "grad_clip": 1.0,30    "batch_size": 32,31    "grad_accum_steps": 1,32    "precision": "f32",33    "checkpoint_every": 500,34    "eval_every": 250,35    "eval_batches": 10,36    "seed": 133737  }38}39