spb/forge Public MIT

1{2  "_comment": "forge-tiny-25m-smoke — same skeleton as forge-tiny-25m but on the existing TinyStories 4k-vocab data, sized for a fast end-to-end validation run (~55 min at ~16k tok/s f32 on an M5 Max). Purpose: prove the full loop (dataloader -> Muon+WSD -> qk_norm -> checkpoint -> resume -> eval -> generate) before spending hours on the real mix. 16 x 8 x 1024 = 131,072 tokens/step x 400 steps = 52.4M tokens (~2.7 epochs of the local 19.1M-token set). Loss should fall well below 3.0; sampled stories must be coherent English. If this run misbehaves, nothing bigger gets launched.",3  "model": {4    "name": "forge-tiny-25m-smoke",5    "n_layers": 8,6    "d_model": 384,7    "n_heads": 6,8    "n_kv_heads": 3,9    "d_ff": 1024,10    "vocab_size": 4096,11    "context_length": 1024,12    "tied_embeddings": true,13    "use_rope": true,14    "rope_theta": 10000.0,15    "norm": "rmsnorm",16    "norm_eps": 1e-06,17    "activation": "swiglu",18    "qk_norm": true,19    "dropout": 0.020  },21  "train": {22    "optimizer": "muon",23    "muon_lr": 0.02,24    "muon_momentum": 0.95,25    "lr": 0.0005,26    "min_lr_ratio": 0.1,27    "schedule": "wsd",28    "wsd_decay_frac": 0.15,29    "warmup_steps": 40,30    "max_steps": 400,31    "beta1": 0.9,32    "beta2": 0.95,33    "eps": 1e-08,34    "weight_decay": 0.1,35    "grad_clip": 1.0,36    "batch_size": 16,37    "grad_accum_steps": 8,38    "precision": "f32",39    "checkpoint_every": 100,40    "eval_every": 100,41    "eval_batches": 20,42    "seed": 133743  }44}45