{
  "_comment": "205.6M params, ONE EPOCH over the full TinyStories corpus on M3U96a (M3 Ultra, 60-core GPU, 96 GB). 8 x 32 x 1024 = 262144 tokens/step x 1553 steps = 407.1M of 407,344,713 tokens (99.94%). Micro-batch 8 halves activation memory vs 16; grad_accum 32 keeps the effective batch (and so the 3e-4 LR) unchanged. precision is parsed but not yet honored - all kernels are f32.",
  "model": {
    "name": "gpt-200m-cluster",
    "n_layers": 16,
    "d_model": 1024,
    "n_heads": 16,
    "n_kv_heads": 8,
    "d_ff": 3072,
    "vocab_size": 4096,
    "context_length": 1024,
    "tied_embeddings": true,
    "use_rope": true,
    "rope_theta": 10000.0,
    "norm": "rmsnorm",
    "norm_eps": 1e-06,
    "activation": "swiglu",
    "dropout": 0.0
  },
  "train": {
    "lr": 0.0003,
    "min_lr_ratio": 0.1,
    "warmup_steps": 155,
    "max_steps": 1553,
    "beta1": 0.9,
    "beta2": 0.95,
    "eps": 1e-08,
    "weight_decay": 0.1,
    "grad_clip": 1.0,
    "batch_size": 8,
    "grad_accum_steps": 32,
    "precision": "f32",
    "checkpoint_every": 100,
    "eval_every": 100,
    "eval_batches": 10,
    "seed": 1337
  }
}