chore: commit pending workspace changes

Simon-Pierre Boucher committed 3 mo ago (Aug 10, 2026) parent 147560e Browse files

2 changed files +82 −0

added configs/forge-tiny-25m-smoke.json +44 −0

++{
++  "_comment": "forge-tiny-25m-smoke — same skeleton as forge-tiny-25m but on the existing TinyStories 4k-vocab data, sized for a fast end-to-end validation run (~55 min at ~16k tok/s f32 on an M5 Max). Purpose: prove the full loop (dataloader -> Muon+WSD -> qk_norm -> checkpoint -> resume -> eval -> generate) before spending hours on the real mix. 16 x 8 x 1024 = 131,072 tokens/step x 400 steps = 52.4M tokens (~2.7 epochs of the local 19.1M-token set). Loss should fall well below 3.0; sampled stories must be coherent English. If this run misbehaves, nothing bigger gets launched.",
++  "model": {
++    "name": "forge-tiny-25m-smoke",
++    "n_layers": 8,
++    "d_model": 384,
++    "n_heads": 6,
++    "n_kv_heads": 3,
++    "d_ff": 1024,
++    "vocab_size": 4096,
++    "context_length": 1024,
++    "tied_embeddings": true,
++    "use_rope": true,
++    "rope_theta": 10000.0,
++    "norm": "rmsnorm",
++    "norm_eps": 1e-06,
++    "activation": "swiglu",
++    "qk_norm": true,
++    "dropout": 0.0
++  },
++  "train": {
++    "optimizer": "muon",
++    "muon_lr": 0.02,
++    "muon_momentum": 0.95,
++    "lr": 0.0005,
++    "min_lr_ratio": 0.1,
++    "schedule": "wsd",
++    "wsd_decay_frac": 0.15,
++    "warmup_steps": 40,
++    "max_steps": 400,
++    "beta1": 0.9,
++    "beta2": 0.95,
++    "eps": 1e-08,
++    "weight_decay": 0.1,
++    "grad_clip": 1.0,
++    "batch_size": 16,
++    "grad_accum_steps": 8,
++    "precision": "f32",
++    "checkpoint_every": 100,
++    "eval_every": 100,
++    "eval_batches": 20,
++    "seed": 1337
++  }
++}

added configs/richard-10m-1epoch.json +38 −0

++{
++  "_comment": "Model 'richard' (10M params) — one epoch over the 19.14M-token TinyStories set: 584 steps x 64 sequences x 512 tokens = 19.1M tokens. Warmup is 10% of the run.",
++  "model": {
++    "name": "richard",
++    "n_layers": 6,
++    "d_model": 384,
++    "n_heads": 6,
++    "n_kv_heads": 6,
++    "d_ff": 1024,
++    "vocab_size": 4096,
++    "context_length": 512,
++    "tied_embeddings": true,
++    "use_rope": true,
++    "rope_theta": 10000.0,
++    "norm": "rmsnorm",
++    "norm_eps": 1e-06,
++    "activation": "swiglu",
++    "dropout": 0.0
++  },
++  "train": {
++    "lr": 0.0006,
++    "min_lr_ratio": 0.1,
++    "warmup_steps": 58,
++    "max_steps": 584,
++    "beta1": 0.9,
++    "beta2": 0.95,
++    "eps": 1e-08,
++    "weight_decay": 0.1,
++    "grad_clip": 1.0,
++    "batch_size": 64,
++    "grad_accum_steps": 1,
++    "precision": "f32",
++    "checkpoint_every": 200,
++    "eval_every": 100,
++    "eval_batches": 20,
++    "seed": 1337
++  }
++}

spb/forge

chore: commit pending workspace changes

Theme