|
1 |
+{ |
|
2 |
+ "_comment": "forge-tiny-25m-smoke — same skeleton as forge-tiny-25m but on the existing TinyStories 4k-vocab data, sized for a fast end-to-end validation run (~55 min at ~16k tok/s f32 on an M5 Max). Purpose: prove the full loop (dataloader -> Muon+WSD -> qk_norm -> checkpoint -> resume -> eval -> generate) before spending hours on the real mix. 16 x 8 x 1024 = 131,072 tokens/step x 400 steps = 52.4M tokens (~2.7 epochs of the local 19.1M-token set). Loss should fall well below 3.0; sampled stories must be coherent English. If this run misbehaves, nothing bigger gets launched.", |
|
3 |
+ "model": { |
|
4 |
+ "name": "forge-tiny-25m-smoke", |
|
5 |
+ "n_layers": 8, |
|
6 |
+ "d_model": 384, |
|
7 |
+ "n_heads": 6, |
|
8 |
+ "n_kv_heads": 3, |
|
9 |
+ "d_ff": 1024, |
|
10 |
+ "vocab_size": 4096, |
|
11 |
+ "context_length": 1024, |
|
12 |
+ "tied_embeddings": true, |
|
13 |
+ "use_rope": true, |
|
14 |
+ "rope_theta": 10000.0, |
|
15 |
+ "norm": "rmsnorm", |
|
16 |
+ "norm_eps": 1e-06, |
|
17 |
+ "activation": "swiglu", |
|
18 |
+ "qk_norm": true, |
|
19 |
+ "dropout": 0.0 |
|
20 |
+ }, |
|
21 |
+ "train": { |
|
22 |
+ "optimizer": "muon", |
|
23 |
+ "muon_lr": 0.02, |
|
24 |
+ "muon_momentum": 0.95, |
|
25 |
+ "lr": 0.0005, |
|
26 |
+ "min_lr_ratio": 0.1, |
|
27 |
+ "schedule": "wsd", |
|
28 |
+ "wsd_decay_frac": 0.15, |
|
29 |
+ "warmup_steps": 40, |
|
30 |
+ "max_steps": 400, |
|
31 |
+ "beta1": 0.9, |
|
32 |
+ "beta2": 0.95, |
|
33 |
+ "eps": 1e-08, |
|
34 |
+ "weight_decay": 0.1, |
|
35 |
+ "grad_clip": 1.0, |
|
36 |
+ "batch_size": 16, |
|
37 |
+ "grad_accum_steps": 8, |
|
38 |
+ "precision": "f32", |
|
39 |
+ "checkpoint_every": 100, |
|
40 |
+ "eval_every": 100, |
|
41 |
+ "eval_batches": 20, |
|
42 |
+ "seed": 1337 |
|
43 |
+ } |
|
44 |
+} |