spb/forge Public MIT

Forge — LLM training from scratch in pure C++20 + Metal on Apple Silicon.

C++ 61.2% C 23% Python 7.6% TeX 7.2% CMake 1.1%

738 B · 38 lines json

Raw Blame History

1{2  "model": {3    "name": "gpt-10m",4    "n_layers": 6,5    "d_model": 384,6    "n_heads": 6,7    "n_kv_heads": 6,8    "d_ff": 1024,9    "vocab_size": 4096,10    "context_length": 512,11    "tied_embeddings": true,12    "use_rope": true,13    "rope_theta": 10000.0,14    "norm": "rmsnorm",15    "norm_eps": 1e-6,16    "activation": "swiglu",17    "dropout": 0.018  },19  "train": {20    "lr": 6e-4,21    "min_lr_ratio": 0.1,22    "warmup_steps": 1000,23    "max_steps": 20000,24    "beta1": 0.9,25    "beta2": 0.95,26    "eps": 1e-8,27    "weight_decay": 0.1,28    "grad_clip": 1.0,29    "batch_size": 64,30    "grad_accum_steps": 1,31    "precision": "f32",32    "checkpoint_every": 1000,33    "eval_every": 500,34    "eval_batches": 20,35    "seed": 133736  }37}38