spb/forge Public MIT

1{2  "_comment": "Gemma-3-style variant of gpt-50m: 5 local (window 256, theta 10k) : 1 global (theta 1M) attention pattern, QK-norm, sandwich norm, embeddings scaled by sqrt(d_model), GELU. Same data/steps as gpt-50m.",3  "model": {4    "name": "gpt-50m-gemma",5    "n_layers": 12,6    "d_model": 576,7    "n_heads": 9,8    "n_kv_heads": 3,9    "d_ff": 1536,10    "vocab_size": 4096,11    "context_length": 1024,12    "tied_embeddings": true,13    "use_rope": true,14    "rope_theta": 10000.0,15    "rope_theta_global": 1000000.0,16    "norm": "rmsnorm",17    "norm_eps": 1e-06,18    "activation": "gelu",19    "dropout": 0.0,20    "qk_norm": true,21    "norm_placement": "sandwich",22    "scale_embeddings": true,23    "sliding_window": 256,24    "sliding_global_every": 625  },26  "train": {27    "lr": 0.0005,28    "min_lr_ratio": 0.1,29    "warmup_steps": 117,30    "max_steps": 1170,31    "beta1": 0.9,32    "beta2": 0.95,33    "eps": 1e-08,34    "weight_decay": 0.1,35    "grad_clip": 1.0,36    "batch_size": 8,37    "grad_accum_steps": 8,38    "precision": "f32",39    "checkpoint_every": 200,40    "eval_every": 100,41    "eval_batches": 20,42    "seed": 133743  }44}45