spb/forge Public MIT

1{2  "_comment": "gpt-50m trained with Muon (NS-orthogonalized momentum on hidden matrices, AdamW on embeddings/head) + WSD schedule (flat plateau, 15% 1-sqrt cooldown). muon_lr 0.02 is the NanoGPT-speedrun default; lr applies to the AdamW group. Same data/steps as gpt-50m for A/B comparison.",3  "model": {4    "name": "gpt-50m-muon",5    "n_layers": 10,6    "d_model": 640,7    "n_heads": 10,8    "n_kv_heads": 10,9    "d_ff": 1728,10    "vocab_size": 4096,11    "context_length": 1024,12    "tied_embeddings": true,13    "use_rope": true,14    "rope_theta": 10000.0,15    "norm": "rmsnorm",16    "norm_eps": 1e-06,17    "activation": "swiglu",18    "dropout": 0.019  },20  "train": {21    "lr": 0.0005,22    "min_lr_ratio": 0.1,23    "warmup_steps": 117,24    "max_steps": 1170,25    "schedule": "wsd",26    "wsd_decay_frac": 0.15,27    "optimizer": "muon",28    "muon_lr": 0.02,29    "muon_momentum": 0.95,30    "beta1": 0.9,31    "beta2": 0.95,32    "eps": 1e-08,33    "weight_decay": 0.1,34    "grad_clip": 1.0,35    "batch_size": 8,36    "grad_accum_steps": 8,37    "precision": "f32",38    "checkpoint_every": 200,39    "eval_every": 100,40    "eval_batches": 20,41    "seed": 133742  }43}44