{
 "config": {
  "_comment": "~52M params, sized between gpt-25m and gpt-100m. vocab 4096 to match the local tok4096 train.bin (19.14M tokens). batch_size is the MICRO-batch: 8 x 8 x 1024 = 65536 tokens/step; 1170 steps = ~4 epochs over the 19.14M-token set. Warmup is 10% of the run. precision is parsed but not yet honored - all kernels are f32.",
  "model": {
   "activation": "swiglu",
   "context_length": 1024,
   "d_ff": 1728,
   "d_model": 640,
   "dropout": 0.0,
   "n_heads": 10,
   "n_kv_heads": 10,
   "n_layers": 10,
   "name": "gpt-50m",
   "norm": "rmsnorm",
   "norm_eps": 1e-06,
   "rope_theta": 10000.0,
   "tied_embeddings": true,
   "use_rope": true,
   "vocab_size": 4096
  },
  "train": {
   "batch_size": 8,
   "beta1": 0.9,
   "beta2": 0.95,
   "checkpoint_every": 200,
   "eps": 1e-08,
   "eval_batches": 20,
   "eval_every": 100,
   "grad_accum_steps": 8,
   "grad_clip": 1.0,
   "lr": 0.0005,
   "max_steps": 1170,
   "min_lr_ratio": 0.1,
   "precision": "f32",
   "seed": 1337,
   "warmup_steps": 117,
   "weight_decay": 0.1
  }
 },
 "dtype": "f32",
 "fmodel": 1,
 "parent": "",
 "self": "manifest-000000.json",
 "step": 1170,
 "tag": "one-epoch-x4",
 "tensors": {
  "blocks.0.attn.wk.weight": {
   "dtype": "f32",
   "hash": "ea0278bff183bd9b",
   "nbytes": 1638400,
   "offset": 12156928,
   "shape": [
    640,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.0.attn.wo.weight": {
   "dtype": "f32",
   "hash": "b08bb6730b3f4047",
   "nbytes": 1638400,
   "offset": 15433728,
   "shape": [
    640,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.0.attn.wq.weight": {
   "dtype": "f32",
   "hash": "c9f0aee32c2afb09",
   "nbytes": 1638400,
   "offset": 10518528,
   "shape": [
    640,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.0.attn.wv.weight": {
   "dtype": "f32",
   "hash": "a46e677dff653afa",
   "nbytes": 1638400,
   "offset": 13795328,
   "shape": [
    640,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.0.mlp.w1.weight": {
   "dtype": "f32",
   "hash": "4b63b198ca18d908",
   "nbytes": 4423680,
   "offset": 17088512,
   "shape": [
    1728,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.0.mlp.w2.weight": {
   "dtype": "f32",
   "hash": "d0b0ced42d5c91d2",
   "nbytes": 4423680,
   "offset": 25935872,
   "shape": [
    640,
    1728
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.0.mlp.w3.weight": {
   "dtype": "f32",
   "hash": "d36f453ea5b7576d",
   "nbytes": 4423680,
   "offset": 21512192,
   "shape": [
    1728,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.0.norm1.weight": {
   "dtype": "f32",
   "hash": "bf60929ceab06890",
   "nbytes": 2560,
   "offset": 10502144,
   "shape": [
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.0.norm2.weight": {
   "dtype": "f32",
   "hash": "bb257feeea6d5b6f",
   "nbytes": 2560,
   "offset": 17072128,
   "shape": [
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.1.attn.wk.weight": {
   "dtype": "f32",
   "hash": "a99d45c6af060836",
   "nbytes": 1638400,
   "offset": 32014336,
   "shape": [
    640,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.1.attn.wo.weight": {
   "dtype": "f32",
   "hash": "4ecc0144607917a7",
   "nbytes": 1638400,
   "offset": 35291136,
   "shape": [
    640,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.1.attn.wq.weight": {
   "dtype": "f32",
   "hash": "5dd4c343a6557550",
   "nbytes": 1638400,
   "offset": 30375936,
   "shape": [
    640,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.1.attn.wv.weight": {
   "dtype": "f32",
   "hash": "0de8eef2c1851cef",
   "nbytes": 1638400,
   "offset": 33652736,
   "shape": [
    640,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.1.mlp.w1.weight": {
   "dtype": "f32",
   "hash": "d8fbfcb43a5a4677",
   "nbytes": 4423680,
   "offset": 36945920,
   "shape": [
    1728,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.1.mlp.w2.weight": {
   "dtype": "f32",
   "hash": "df86dbe751fd0897",
   "nbytes": 4423680,
   "offset": 45793280,
   "shape": [
    640,
    1728
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.1.mlp.w3.weight": {
   "dtype": "f32",
   "hash": "e206903aafcacc9a",
   "nbytes": 4423680,
   "offset": 41369600,
   "shape": [
    1728,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.1.norm1.weight": {
   "dtype": "f32",
   "hash": "bc7c7f4144e6c4a0",
   "nbytes": 2560,
   "offset": 30359552,
   "shape": [
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.1.norm2.weight": {
   "dtype": "f32",
   "hash": "1d631908e97d0ab2",
   "nbytes": 2560,
   "offset": 36929536,
   "shape": [
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.2.attn.wk.weight": {
   "dtype": "f32",
   "hash": "e5f3bfa24ce66850",
   "nbytes": 1638400,
   "offset": 51871744,
   "shape": [
    640,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.2.attn.wo.weight": {
   "dtype": "f32",
   "hash": "81e09028c8d4fa64",
   "nbytes": 1638400,
   "offset": 55148544,
   "shape": [
    640,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.2.attn.wq.weight": {
   "dtype": "f32",
   "hash": "70ee42db3250d15e",
   "nbytes": 1638400,
   "offset": 50233344,
   "shape": [
    640,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.2.attn.wv.weight": {
   "dtype": "f32",
   "hash": "b27d667f5ce35172",
   "nbytes": 1638400,
   "offset": 53510144,
   "shape": [
    640,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.2.mlp.w1.weight": {
   "dtype": "f32",
   "hash": "eef179f1c68e1af2",
   "nbytes": 4423680,
   "offset": 56803328,
   "shape": [
    1728,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.2.mlp.w2.weight": {
   "dtype": "f32",
   "hash": "0393a401c220c511",
   "nbytes": 4423680,
   "offset": 65650688,
   "shape": [
    640,
    1728
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.2.mlp.w3.weight": {
   "dtype": "f32",
   "hash": "8b9edaa2ac9409e3",
   "nbytes": 4423680,
   "offset": 61227008,
   "shape": [
    1728,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.2.norm1.weight": {
   "dtype": "f32",
   "hash": "fb9413da5028b6fd",
   "nbytes": 2560,
   "offset": 50216960,
   "shape": [
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.2.norm2.weight": {
   "dtype": "f32",
   "hash": "a76041b595744213",
   "nbytes": 2560,
   "offset": 56786944,
   "shape": [
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.3.attn.wk.weight": {
   "dtype": "f32",
   "hash": "01edcec7013dc445",
   "nbytes": 1638400,
   "offset": 71729152,
   "shape": [
    640,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.3.attn.wo.weight": {
   "dtype": "f32",
   "hash": "ea9891714020fcf4",
   "nbytes": 1638400,
   "offset": 75005952,
   "shape": [
    640,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.3.attn.wq.weight": {
   "dtype": "f32",
   "hash": "8bcfa49d11bf8160",
   "nbytes": 1638400,
   "offset": 70090752,
   "shape": [
    640,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.3.attn.wv.weight": {
   "dtype": "f32",
   "hash": "8a527c0649e24cd9",
   "nbytes": 1638400,
   "offset": 73367552,
   "shape": [
    640,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.3.mlp.w1.weight": {
   "dtype": "f32",
   "hash": "81058be641f54d7a",
   "nbytes": 4423680,
   "offset": 76660736,
   "shape": [
    1728,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.3.mlp.w2.weight": {
   "dtype": "f32",
   "hash": "67d4b0352f96a0f7",
   "nbytes": 4423680,
   "offset": 85508096,
   "shape": [
    640,
    1728
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.3.mlp.w3.weight": {
   "dtype": "f32",
   "hash": "7299e20a40b2da6c",
   "nbytes": 4423680,
   "offset": 81084416,
   "shape": [
    1728,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.3.norm1.weight": {
   "dtype": "f32",
   "hash": "8247bcdfab7adad7",
   "nbytes": 2560,
   "offset": 70074368,
   "shape": [
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.3.norm2.weight": {
   "dtype": "f32",
   "hash": "596b0bb7a9ad16ad",
   "nbytes": 2560,
   "offset": 76644352,
   "shape": [
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.4.attn.wk.weight": {
   "dtype": "f32",
   "hash": "01a11bb4643c874d",
   "nbytes": 1638400,
   "offset": 91586560,
   "shape": [
    640,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.4.attn.wo.weight": {
   "dtype": "f32",
   "hash": "a1e3afb60265703c",
   "nbytes": 1638400,
   "offset": 94863360,
   "shape": [
    640,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.4.attn.wq.weight": {
   "dtype": "f32",
   "hash": "b80af30751ce8923",
   "nbytes": 1638400,
   "offset": 89948160,
   "shape": [
    640,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.4.attn.wv.weight": {
   "dtype": "f32",
   "hash": "6ffdec20fca77461",
   "nbytes": 1638400,
   "offset": 93224960,
   "shape": [
    640,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.4.mlp.w1.weight": {
   "dtype": "f32",
   "hash": "164de67faf3c3be9",
   "nbytes": 4423680,
   "offset": 16384,
   "shape": [
    1728,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.4.mlp.w2.weight": {
   "dtype": "f32",
   "hash": "b95146cee5c723ff",
   "nbytes": 4423680,
   "offset": 8863744,
   "shape": [
    640,
    1728
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.4.mlp.w3.weight": {
   "dtype": "f32",
   "hash": "14b17e17afb81c3c",
   "nbytes": 4423680,
   "offset": 4440064,
   "shape": [
    1728,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.4.norm1.weight": {
   "dtype": "f32",
   "hash": "dddf2bd3cd03a357",
   "nbytes": 2560,
   "offset": 89931776,
   "shape": [
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.4.norm2.weight": {
   "dtype": "f32",
   "hash": "3eac1a00123afade",
   "nbytes": 2560,
   "offset": 96501760,
   "shape": [
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  },
  "blocks.5.attn.wk.weight": {
   "dtype": "f32",
   "hash": "8b994bd11cc5c235",
   "nbytes": 1638400,
   "offset": 14942208,
   "shape": [
    640,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.5.attn.wo.weight": {
   "dtype": "f32",
   "hash": "14d9dff05dd948e5",
   "nbytes": 1638400,
   "offset": 18219008,
   "shape": [
    640,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.5.attn.wq.weight": {
   "dtype": "f32",
   "hash": "da8ef005898a1637",
   "nbytes": 1638400,
   "offset": 13303808,
   "shape": [
    640,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.5.attn.wv.weight": {
   "dtype": "f32",
   "hash": "6b28b2be70bf033a",
   "nbytes": 1638400,
   "offset": 16580608,
   "shape": [
    640,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.5.mlp.w1.weight": {
   "dtype": "f32",
   "hash": "500fae8fdecb95ca",
   "nbytes": 4423680,
   "offset": 19873792,
   "shape": [
    1728,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.5.mlp.w2.weight": {
   "dtype": "f32",
   "hash": "6b84ccc84ca6b6fa",
   "nbytes": 4423680,
   "offset": 28721152,
   "shape": [
    640,
    1728
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.5.mlp.w3.weight": {
   "dtype": "f32",
   "hash": "541490e9a2d38329",
   "nbytes": 4423680,
   "offset": 24297472,
   "shape": [
    1728,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.5.norm1.weight": {
   "dtype": "f32",
   "hash": "9712111d7db8b749",
   "nbytes": 2560,
   "offset": 13287424,
   "shape": [
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.5.norm2.weight": {
   "dtype": "f32",
   "hash": "9656dd83e2703195",
   "nbytes": 2560,
   "offset": 19857408,
   "shape": [
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.6.attn.wk.weight": {
   "dtype": "f32",
   "hash": "46d6a33811ae7958",
   "nbytes": 1638400,
   "offset": 34799616,
   "shape": [
    640,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.6.attn.wo.weight": {
   "dtype": "f32",
   "hash": "0af48008a7e9d48d",
   "nbytes": 1638400,
   "offset": 38076416,
   "shape": [
    640,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.6.attn.wq.weight": {
   "dtype": "f32",
   "hash": "6f4660a328d4b423",
   "nbytes": 1638400,
   "offset": 33161216,
   "shape": [
    640,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.6.attn.wv.weight": {
   "dtype": "f32",
   "hash": "2b22a3c36d0fa06d",
   "nbytes": 1638400,
   "offset": 36438016,
   "shape": [
    640,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.6.mlp.w1.weight": {
   "dtype": "f32",
   "hash": "9ffa6ede45d79cb6",
   "nbytes": 4423680,
   "offset": 39731200,
   "shape": [
    1728,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.6.mlp.w2.weight": {
   "dtype": "f32",
   "hash": "cded5100040c6a61",
   "nbytes": 4423680,
   "offset": 48578560,
   "shape": [
    640,
    1728
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.6.mlp.w3.weight": {
   "dtype": "f32",
   "hash": "bd923d3a8f5162e6",
   "nbytes": 4423680,
   "offset": 44154880,
   "shape": [
    1728,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.6.norm1.weight": {
   "dtype": "f32",
   "hash": "063bfae2ccd1b891",
   "nbytes": 2560,
   "offset": 33144832,
   "shape": [
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.6.norm2.weight": {
   "dtype": "f32",
   "hash": "3c20ba70fba95945",
   "nbytes": 2560,
   "offset": 39714816,
   "shape": [
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.7.attn.wk.weight": {
   "dtype": "f32",
   "hash": "52ea38d02ac4ae3e",
   "nbytes": 1638400,
   "offset": 54657024,
   "shape": [
    640,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.7.attn.wo.weight": {
   "dtype": "f32",
   "hash": "945ecf9b46aeb099",
   "nbytes": 1638400,
   "offset": 57933824,
   "shape": [
    640,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.7.attn.wq.weight": {
   "dtype": "f32",
   "hash": "bb4f532e47bbb588",
   "nbytes": 1638400,
   "offset": 53018624,
   "shape": [
    640,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.7.attn.wv.weight": {
   "dtype": "f32",
   "hash": "39fc5bb55f990e08",
   "nbytes": 1638400,
   "offset": 56295424,
   "shape": [
    640,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.7.mlp.w1.weight": {
   "dtype": "f32",
   "hash": "661591194873739a",
   "nbytes": 4423680,
   "offset": 59588608,
   "shape": [
    1728,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.7.mlp.w2.weight": {
   "dtype": "f32",
   "hash": "c0693bdee8cb26f0",
   "nbytes": 4423680,
   "offset": 68435968,
   "shape": [
    640,
    1728
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.7.mlp.w3.weight": {
   "dtype": "f32",
   "hash": "c46311e050bbc2e7",
   "nbytes": 4423680,
   "offset": 64012288,
   "shape": [
    1728,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.7.norm1.weight": {
   "dtype": "f32",
   "hash": "31ddf245176231e8",
   "nbytes": 2560,
   "offset": 53002240,
   "shape": [
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.7.norm2.weight": {
   "dtype": "f32",
   "hash": "adb0020bde0202ae",
   "nbytes": 2560,
   "offset": 59572224,
   "shape": [
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.8.attn.wk.weight": {
   "dtype": "f32",
   "hash": "8b49975f1bcab4b1",
   "nbytes": 1638400,
   "offset": 74514432,
   "shape": [
    640,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.8.attn.wo.weight": {
   "dtype": "f32",
   "hash": "d0fd8ea0a57aaf34",
   "nbytes": 1638400,
   "offset": 77791232,
   "shape": [
    640,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.8.attn.wq.weight": {
   "dtype": "f32",
   "hash": "be6bf060be0ceb6a",
   "nbytes": 1638400,
   "offset": 72876032,
   "shape": [
    640,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.8.attn.wv.weight": {
   "dtype": "f32",
   "hash": "876779888157dd34",
   "nbytes": 1638400,
   "offset": 76152832,
   "shape": [
    640,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.8.mlp.w1.weight": {
   "dtype": "f32",
   "hash": "551d9ffdddc55001",
   "nbytes": 4423680,
   "offset": 79446016,
   "shape": [
    1728,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.8.mlp.w2.weight": {
   "dtype": "f32",
   "hash": "710446ba331dbb33",
   "nbytes": 4423680,
   "offset": 88293376,
   "shape": [
    640,
    1728
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.8.mlp.w3.weight": {
   "dtype": "f32",
   "hash": "75404c4f6c52ca7b",
   "nbytes": 4423680,
   "offset": 83869696,
   "shape": [
    1728,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.8.norm1.weight": {
   "dtype": "f32",
   "hash": "a67c53d98e996f4d",
   "nbytes": 2560,
   "offset": 72859648,
   "shape": [
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.8.norm2.weight": {
   "dtype": "f32",
   "hash": "b7cc1cd235754b8d",
   "nbytes": 2560,
   "offset": 79429632,
   "shape": [
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.9.attn.wk.weight": {
   "dtype": "f32",
   "hash": "77ca114cacd59f92",
   "nbytes": 1638400,
   "offset": 94371840,
   "shape": [
    640,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.9.attn.wo.weight": {
   "dtype": "f32",
   "hash": "2bd03b1679da1e59",
   "nbytes": 1638400,
   "offset": 97648640,
   "shape": [
    640,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.9.attn.wq.weight": {
   "dtype": "f32",
   "hash": "c4ec432807c49aea",
   "nbytes": 1638400,
   "offset": 92733440,
   "shape": [
    640,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.9.attn.wv.weight": {
   "dtype": "f32",
   "hash": "4f2bdf517e8aa817",
   "nbytes": 1638400,
   "offset": 96010240,
   "shape": [
    640,
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.9.mlp.w1.weight": {
   "dtype": "f32",
   "hash": "f65ccd0b671cebed",
   "nbytes": 4423680,
   "offset": 16384,
   "shape": [
    1728,
    640
   ],
   "shard": "objects/24a9ba4df0fd71ed.fshard"
  },
  "blocks.9.mlp.w2.weight": {
   "dtype": "f32",
   "hash": "14f2da31880d173e",
   "nbytes": 4423680,
   "offset": 8863744,
   "shape": [
    640,
    1728
   ],
   "shard": "objects/24a9ba4df0fd71ed.fshard"
  },
  "blocks.9.mlp.w3.weight": {
   "dtype": "f32",
   "hash": "08871f04b2cb1bb0",
   "nbytes": 4423680,
   "offset": 4440064,
   "shape": [
    1728,
    640
   ],
   "shard": "objects/24a9ba4df0fd71ed.fshard"
  },
  "blocks.9.norm1.weight": {
   "dtype": "f32",
   "hash": "ce6abe2627974147",
   "nbytes": 2560,
   "offset": 92717056,
   "shape": [
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "blocks.9.norm2.weight": {
   "dtype": "f32",
   "hash": "cee537f25da157b9",
   "nbytes": 2560,
   "offset": 99287040,
   "shape": [
    640
   ],
   "shard": "objects/b4e0145ce1f2ff18.fshard"
  },
  "final_norm.weight": {
   "dtype": "f32",
   "hash": "e14ca576c57ef871",
   "nbytes": 2560,
   "offset": 13287424,
   "shape": [
    640
   ],
   "shard": "objects/24a9ba4df0fd71ed.fshard"
  },
  "tok_emb.weight": {
   "dtype": "f32",
   "hash": "1d963329140f196a",
   "nbytes": 10485760,
   "offset": 16384,
   "shape": [
    4096,
    640
   ],
   "shard": "objects/72626eb74479f8dd.fshard"
  }
 }
}