spb/forge Public MIT

1{2  "_comment": "Mistral-style variant of gpt-50m: sliding-window attention (256 tokens, every layer), GQA 2:1, QKV bias off. Same data/steps as gpt-50m so the only variable is the attention pattern. The fused scalar flash kernel skips out-of-window KV blocks, so attention cost scales with the window, not the context.",3  "model": {4    "name": "gpt-50m-mistral",5    "n_layers": 10,6    "d_model": 640,7    "n_heads": 10,8    "n_kv_heads": 5,9    "d_ff": 1728,10    "vocab_size": 4096,11    "context_length": 1024,12    "tied_embeddings": true,13    "use_rope": true,14    "rope_theta": 10000.0,15    "norm": "rmsnorm",16    "norm_eps": 1e-06,17    "activation": "swiglu",18    "dropout": 0.0,19    "sliding_window": 25620  },21  "train": {22    "lr": 0.0005,23    "min_lr_ratio": 0.1,24    "warmup_steps": 117,25    "max_steps": 1170,26    "beta1": 0.9,27    "beta2": 0.95,28    "eps": 1e-08,29    "weight_decay": 0.1,30    "grad_clip": 1.0,31    "batch_size": 8,32    "grad_accum_steps": 8,33    "precision": "f32",34    "checkpoint_every": 200,35    "eval_every": 100,36    "eval_batches": 20,37    "seed": 133738  }39}40