spb/forge Public MIT
Forge — LLM training from scratch in pure C++20 + Metal on Apple Silicon.
C++ 61.2%
C 23%
Python 7.6%
TeX 7.2%
CMake 1.1%
1{2 "config": {3 "_comment": "~52M params, sized between gpt-25m and gpt-100m. vocab 4096 to match the local tok4096 train.bin (19.14M tokens). batch_size is the MICRO-batch: 8 x 8 x 1024 = 65536 tokens/step; 1170 steps = ~4 epochs over the 19.14M-token set. Warmup is 10% of the run. precision is parsed but not yet honored - all kernels are f32.",4 "model": {5 "activation": "swiglu",6 "context_length": 1024,7 "d_ff": 1728,8 "d_model": 640,9 "dropout": 0.0,10 "n_heads": 10,11 "n_kv_heads": 10,12 "n_layers": 10,13 "name": "gpt-50m",14 "norm": "rmsnorm",15 "norm_eps": 1e-06,16 "rope_theta": 10000.0,17 "tied_embeddings": true,18 "use_rope": true,19 "vocab_size": 409620 },21 "train": {22 "batch_size": 8,23 "beta1": 0.9,24 "beta2": 0.95,25 "checkpoint_every": 200,26 "eps": 1e-08,27 "eval_batches": 20,28 "eval_every": 100,29 "grad_accum_steps": 8,30 "grad_clip": 1.0,31 "lr": 0.0005,32 "max_steps": 1170,33 "min_lr_ratio": 0.1,34 "precision": "f32",35 "seed": 1337,36 "warmup_steps": 117,37 "weight_decay": 0.138 }39 },40 "dtype": "f32",41 "fmodel": 1,42 "parent": "",43 "self": "manifest-000000.json",44 "step": 1170,45 "tag": "one-epoch-x4",46 "tensors": {47 "blocks.0.attn.wk.weight": {48 "dtype": "f32",49 "hash": "ea0278bff183bd9b",50 "nbytes": 1638400,51 "offset": 12156928,52 "shape": [53 640,54 64055 ],56 "shard": "objects/72626eb74479f8dd.fshard"57 },58 "blocks.0.attn.wo.weight": {59 "dtype": "f32",60 "hash": "b08bb6730b3f4047",61 "nbytes": 1638400,62 "offset": 15433728,63 "shape": [64 640,65 64066 ],67 "shard": "objects/72626eb74479f8dd.fshard"68 },69 "blocks.0.attn.wq.weight": {70 "dtype": "f32",71 "hash": "c9f0aee32c2afb09",72 "nbytes": 1638400,73 "offset": 10518528,74 "shape": [75 640,76 64077 ],78 "shard": "objects/72626eb74479f8dd.fshard"79 },80 "blocks.0.attn.wv.weight": {81 "dtype": "f32",82 "hash": "a46e677dff653afa",83 "nbytes": 1638400,84 "offset": 13795328,85 "shape": [86 640,87 64088 ],89 "shard": "objects/72626eb74479f8dd.fshard"90 },91 "blocks.0.mlp.w1.weight": {92 "dtype": "f32",93 "hash": "4b63b198ca18d908",94 "nbytes": 4423680,95 "offset": 17088512,96 "shape": [97 1728,98 64099 ],100 "shard": "objects/72626eb74479f8dd.fshard"101 },102 "blocks.0.mlp.w2.weight": {103 "dtype": "f32",104 "hash": "d0b0ced42d5c91d2",105 "nbytes": 4423680,106 "offset": 25935872,107 "shape": [108 640,109 1728110 ],111 "shard": "objects/72626eb74479f8dd.fshard"112 },113 "blocks.0.mlp.w3.weight": {114 "dtype": "f32",115 "hash": "d36f453ea5b7576d",116 "nbytes": 4423680,117 "offset": 21512192,118 "shape": [119 1728,120 640121 ],122 "shard": "objects/72626eb74479f8dd.fshard"123 },124 "blocks.0.norm1.weight": {125 "dtype": "f32",126 "hash": "bf60929ceab06890",127 "nbytes": 2560,128 "offset": 10502144,129 "shape": [130 640131 ],132 "shard": "objects/72626eb74479f8dd.fshard"133 },134 "blocks.0.norm2.weight": {135 "dtype": "f32",136 "hash": "bb257feeea6d5b6f",137 "nbytes": 2560,138 "offset": 17072128,139 "shape": [140 640141 ],142 "shard": "objects/72626eb74479f8dd.fshard"143 },144 "blocks.1.attn.wk.weight": {145 "dtype": "f32",146 "hash": "a99d45c6af060836",147 "nbytes": 1638400,148 "offset": 32014336,149 "shape": [150 640,151 640152 ],153 "shard": "objects/72626eb74479f8dd.fshard"154 },155 "blocks.1.attn.wo.weight": {156 "dtype": "f32",157 "hash": "4ecc0144607917a7",158 "nbytes": 1638400,159 "offset": 35291136,160 "shape": [161 640,162 640163 ],164 "shard": "objects/72626eb74479f8dd.fshard"165 },166 "blocks.1.attn.wq.weight": {167 "dtype": "f32",168 "hash": "5dd4c343a6557550",169 "nbytes": 1638400,170 "offset": 30375936,171 "shape": [172 640,173 640174 ],175 "shard": "objects/72626eb74479f8dd.fshard"176 },177 "blocks.1.attn.wv.weight": {178 "dtype": "f32",179 "hash": "0de8eef2c1851cef",180 "nbytes": 1638400,181 "offset": 33652736,182 "shape": [183 640,184 640185 ],186 "shard": "objects/72626eb74479f8dd.fshard"187 },188 "blocks.1.mlp.w1.weight": {189 "dtype": "f32",190 "hash": "d8fbfcb43a5a4677",191 "nbytes": 4423680,192 "offset": 36945920,193 "shape": [194 1728,195 640196 ],197 "shard": "objects/72626eb74479f8dd.fshard"198 },199 "blocks.1.mlp.w2.weight": {200 "dtype": "f32",201 "hash": "df86dbe751fd0897",202 "nbytes": 4423680,203 "offset": 45793280,204 "shape": [205 640,206 1728207 ],208 "shard": "objects/72626eb74479f8dd.fshard"209 },210 "blocks.1.mlp.w3.weight": {211 "dtype": "f32",212 "hash": "e206903aafcacc9a",213 "nbytes": 4423680,214 "offset": 41369600,215 "shape": [216 1728,217 640218 ],219 "shard": "objects/72626eb74479f8dd.fshard"220 },221 "blocks.1.norm1.weight": {222 "dtype": "f32",223 "hash": "bc7c7f4144e6c4a0",224 "nbytes": 2560,225 "offset": 30359552,226 "shape": [227 640228 ],229 "shard": "objects/72626eb74479f8dd.fshard"230 },231 "blocks.1.norm2.weight": {232 "dtype": "f32",233 "hash": "1d631908e97d0ab2",234 "nbytes": 2560,235 "offset": 36929536,236 "shape": [237 640238 ],239 "shard": "objects/72626eb74479f8dd.fshard"240 },241 "blocks.2.attn.wk.weight": {242 "dtype": "f32",243 "hash": "e5f3bfa24ce66850",244 "nbytes": 1638400,245 "offset": 51871744,246 "shape": [247 640,248 640249 ],250 "shard": "objects/72626eb74479f8dd.fshard"251 },252 "blocks.2.attn.wo.weight": {253 "dtype": "f32",254 "hash": "81e09028c8d4fa64",255 "nbytes": 1638400,256 "offset": 55148544,257 "shape": [258 640,259 640260 ],261 "shard": "objects/72626eb74479f8dd.fshard"262 },263 "blocks.2.attn.wq.weight": {264 "dtype": "f32",265 "hash": "70ee42db3250d15e",266 "nbytes": 1638400,267 "offset": 50233344,268 "shape": [269 640,270 640271 ],272 "shard": "objects/72626eb74479f8dd.fshard"273 },274 "blocks.2.attn.wv.weight": {275 "dtype": "f32",276 "hash": "b27d667f5ce35172",277 "nbytes": 1638400,278 "offset": 53510144,279 "shape": [280 640,281 640282 ],283 "shard": "objects/72626eb74479f8dd.fshard"284 },285 "blocks.2.mlp.w1.weight": {286 "dtype": "f32",287 "hash": "eef179f1c68e1af2",288 "nbytes": 4423680,289 "offset": 56803328,290 "shape": [291 1728,292 640293 ],294 "shard": "objects/72626eb74479f8dd.fshard"295 },296 "blocks.2.mlp.w2.weight": {297 "dtype": "f32",298 "hash": "0393a401c220c511",299 "nbytes": 4423680,300 "offset": 65650688,301 "shape": [302 640,303 1728304 ],305 "shard": "objects/72626eb74479f8dd.fshard"306 },307 "blocks.2.mlp.w3.weight": {308 "dtype": "f32",309 "hash": "8b9edaa2ac9409e3",310 "nbytes": 4423680,311 "offset": 61227008,312 "shape": [313 1728,314 640315 ],316 "shard": "objects/72626eb74479f8dd.fshard"317 },318 "blocks.2.norm1.weight": {319 "dtype": "f32",320 "hash": "fb9413da5028b6fd",321 "nbytes": 2560,322 "offset": 50216960,323 "shape": [324 640325 ],326 "shard": "objects/72626eb74479f8dd.fshard"327 },328 "blocks.2.norm2.weight": {329 "dtype": "f32",330 "hash": "a76041b595744213",331 "nbytes": 2560,332 "offset": 56786944,333 "shape": [334 640335 ],336 "shard": "objects/72626eb74479f8dd.fshard"337 },338 "blocks.3.attn.wk.weight": {339 "dtype": "f32",340 "hash": "01edcec7013dc445",341 "nbytes": 1638400,342 "offset": 71729152,343 "shape": [344 640,345 640346 ],347 "shard": "objects/72626eb74479f8dd.fshard"348 },349 "blocks.3.attn.wo.weight": {350 "dtype": "f32",351 "hash": "ea9891714020fcf4",352 "nbytes": 1638400,353 "offset": 75005952,354 "shape": [355 640,356 640357 ],358 "shard": "objects/72626eb74479f8dd.fshard"359 },360 "blocks.3.attn.wq.weight": {361 "dtype": "f32",362 "hash": "8bcfa49d11bf8160",363 "nbytes": 1638400,364 "offset": 70090752,365 "shape": [366 640,367 640368 ],369 "shard": "objects/72626eb74479f8dd.fshard"370 },371 "blocks.3.attn.wv.weight": {372 "dtype": "f32",373 "hash": "8a527c0649e24cd9",374 "nbytes": 1638400,375 "offset": 73367552,376 "shape": [377 640,378 640379 ],380 "shard": "objects/72626eb74479f8dd.fshard"381 },382 "blocks.3.mlp.w1.weight": {383 "dtype": "f32",384 "hash": "81058be641f54d7a",385 "nbytes": 4423680,386 "offset": 76660736,387 "shape": [388 1728,389 640390 ],391 "shard": "objects/72626eb74479f8dd.fshard"392 },393 "blocks.3.mlp.w2.weight": {394 "dtype": "f32",395 "hash": "67d4b0352f96a0f7",396 "nbytes": 4423680,397 "offset": 85508096,398 "shape": [399 640,400 1728401 ],402 "shard": "objects/72626eb74479f8dd.fshard"403 },404 "blocks.3.mlp.w3.weight": {405 "dtype": "f32",406 "hash": "7299e20a40b2da6c",407 "nbytes": 4423680,408 "offset": 81084416,409 "shape": [410 1728,411 640412 ],413 "shard": "objects/72626eb74479f8dd.fshard"414 },415 "blocks.3.norm1.weight": {416 "dtype": "f32",417 "hash": "8247bcdfab7adad7",418 "nbytes": 2560,419 "offset": 70074368,420 "shape": [421 640422 ],423 "shard": "objects/72626eb74479f8dd.fshard"424 },425 "blocks.3.norm2.weight": {426 "dtype": "f32",427 "hash": "596b0bb7a9ad16ad",428 "nbytes": 2560,429 "offset": 76644352,430 "shape": [431 640432 ],433 "shard": "objects/72626eb74479f8dd.fshard"434 },435 "blocks.4.attn.wk.weight": {436 "dtype": "f32",437 "hash": "01a11bb4643c874d",438 "nbytes": 1638400,439 "offset": 91586560,440 "shape": [441 640,442 640443 ],444 "shard": "objects/72626eb74479f8dd.fshard"445 },446 "blocks.4.attn.wo.weight": {447 "dtype": "f32",448 "hash": "a1e3afb60265703c",449 "nbytes": 1638400,450 "offset": 94863360,451 "shape": [452 640,453 640454 ],455 "shard": "objects/72626eb74479f8dd.fshard"456 },457 "blocks.4.attn.wq.weight": {458 "dtype": "f32",459 "hash": "b80af30751ce8923",460 "nbytes": 1638400,461 "offset": 89948160,462 "shape": [463 640,464 640465 ],466 "shard": "objects/72626eb74479f8dd.fshard"467 },468 "blocks.4.attn.wv.weight": {469 "dtype": "f32",470 "hash": "6ffdec20fca77461",471 "nbytes": 1638400,472 "offset": 93224960,473 "shape": [474 640,475 640476 ],477 "shard": "objects/72626eb74479f8dd.fshard"478 },479 "blocks.4.mlp.w1.weight": {480 "dtype": "f32",481 "hash": "164de67faf3c3be9",482 "nbytes": 4423680,483 "offset": 16384,484 "shape": [485 1728,486 640487 ],488 "shard": "objects/b4e0145ce1f2ff18.fshard"489 },490 "blocks.4.mlp.w2.weight": {491 "dtype": "f32",492 "hash": "b95146cee5c723ff",493 "nbytes": 4423680,494 "offset": 8863744,495 "shape": [496 640,497 1728498 ],499 "shard": "objects/b4e0145ce1f2ff18.fshard"500 },501 "blocks.4.mlp.w3.weight": {502 "dtype": "f32",503 "hash": "14b17e17afb81c3c",504 "nbytes": 4423680,505 "offset": 4440064,506 "shape": [507 1728,508 640509 ],510 "shard": "objects/b4e0145ce1f2ff18.fshard"511 },512 "blocks.4.norm1.weight": {513 "dtype": "f32",514 "hash": "dddf2bd3cd03a357",515 "nbytes": 2560,516 "offset": 89931776,517 "shape": [518 640519 ],520 "shard": "objects/72626eb74479f8dd.fshard"521 },522 "blocks.4.norm2.weight": {523 "dtype": "f32",524 "hash": "3eac1a00123afade",525 "nbytes": 2560,526 "offset": 96501760,527 "shape": [528 640529 ],530 "shard": "objects/72626eb74479f8dd.fshard"531 },532 "blocks.5.attn.wk.weight": {533 "dtype": "f32",534 "hash": "8b994bd11cc5c235",535 "nbytes": 1638400,536 "offset": 14942208,537 "shape": [538 640,539 640540 ],541 "shard": "objects/b4e0145ce1f2ff18.fshard"542 },543 "blocks.5.attn.wo.weight": {544 "dtype": "f32",545 "hash": "14d9dff05dd948e5",546 "nbytes": 1638400,547 "offset": 18219008,548 "shape": [549 640,550 640551 ],552 "shard": "objects/b4e0145ce1f2ff18.fshard"553 },554 "blocks.5.attn.wq.weight": {555 "dtype": "f32",556 "hash": "da8ef005898a1637",557 "nbytes": 1638400,558 "offset": 13303808,559 "shape": [560 640,561 640562 ],563 "shard": "objects/b4e0145ce1f2ff18.fshard"564 },565 "blocks.5.attn.wv.weight": {566 "dtype": "f32",567 "hash": "6b28b2be70bf033a",568 "nbytes": 1638400,569 "offset": 16580608,570 "shape": [571 640,572 640573 ],574 "shard": "objects/b4e0145ce1f2ff18.fshard"575 },576 "blocks.5.mlp.w1.weight": {577 "dtype": "f32",578 "hash": "500fae8fdecb95ca",579 "nbytes": 4423680,580 "offset": 19873792,581 "shape": [582 1728,583 640584 ],585 "shard": "objects/b4e0145ce1f2ff18.fshard"586 },587 "blocks.5.mlp.w2.weight": {588 "dtype": "f32",589 "hash": "6b84ccc84ca6b6fa",590 "nbytes": 4423680,591 "offset": 28721152,592 "shape": [593 640,594 1728595 ],596 "shard": "objects/b4e0145ce1f2ff18.fshard"597 },598 "blocks.5.mlp.w3.weight": {599 "dtype": "f32",600 "hash": "541490e9a2d38329",601 "nbytes": 4423680,602 "offset": 24297472,603 "shape": [604 1728,605 640606 ],607 "shard": "objects/b4e0145ce1f2ff18.fshard"608 },609 "blocks.5.norm1.weight": {610 "dtype": "f32",611 "hash": "9712111d7db8b749",612 "nbytes": 2560,613 "offset": 13287424,614 "shape": [615 640616 ],617 "shard": "objects/b4e0145ce1f2ff18.fshard"618 },619 "blocks.5.norm2.weight": {620 "dtype": "f32",621 "hash": "9656dd83e2703195",622 "nbytes": 2560,623 "offset": 19857408,624 "shape": [625 640626 ],627 "shard": "objects/b4e0145ce1f2ff18.fshard"628 },629 "blocks.6.attn.wk.weight": {630 "dtype": "f32",631 "hash": "46d6a33811ae7958",632 "nbytes": 1638400,633 "offset": 34799616,634 "shape": [635 640,636 640637 ],638 "shard": "objects/b4e0145ce1f2ff18.fshard"639 },640 "blocks.6.attn.wo.weight": {641 "dtype": "f32",642 "hash": "0af48008a7e9d48d",643 "nbytes": 1638400,644 "offset": 38076416,645 "shape": [646 640,647 640648 ],649 "shard": "objects/b4e0145ce1f2ff18.fshard"650 },651 "blocks.6.attn.wq.weight": {652 "dtype": "f32",653 "hash": "6f4660a328d4b423",654 "nbytes": 1638400,655 "offset": 33161216,656 "shape": [657 640,658 640659 ],660 "shard": "objects/b4e0145ce1f2ff18.fshard"661 },662 "blocks.6.attn.wv.weight": {663 "dtype": "f32",664 "hash": "2b22a3c36d0fa06d",665 "nbytes": 1638400,666 "offset": 36438016,667 "shape": [668 640,669 640670 ],671 "shard": "objects/b4e0145ce1f2ff18.fshard"672 },673 "blocks.6.mlp.w1.weight": {674 "dtype": "f32",675 "hash": "9ffa6ede45d79cb6",676 "nbytes": 4423680,677 "offset": 39731200,678 "shape": [679 1728,680 640681 ],682 "shard": "objects/b4e0145ce1f2ff18.fshard"683 },684 "blocks.6.mlp.w2.weight": {685 "dtype": "f32",686 "hash": "cded5100040c6a61",687 "nbytes": 4423680,688 "offset": 48578560,689 "shape": [690 640,691 1728692 ],693 "shard": "objects/b4e0145ce1f2ff18.fshard"694 },695 "blocks.6.mlp.w3.weight": {696 "dtype": "f32",697 "hash": "bd923d3a8f5162e6",698 "nbytes": 4423680,699 "offset": 44154880,700 "shape": [701 1728,702 640703 ],704 "shard": "objects/b4e0145ce1f2ff18.fshard"705 },706 "blocks.6.norm1.weight": {707 "dtype": "f32",708 "hash": "063bfae2ccd1b891",709 "nbytes": 2560,710 "offset": 33144832,711 "shape": [712 640713 ],714 "shard": "objects/b4e0145ce1f2ff18.fshard"715 },716 "blocks.6.norm2.weight": {717 "dtype": "f32",718 "hash": "3c20ba70fba95945",719 "nbytes": 2560,720 "offset": 39714816,721 "shape": [722 640723 ],724 "shard": "objects/b4e0145ce1f2ff18.fshard"725 },726 "blocks.7.attn.wk.weight": {727 "dtype": "f32",728 "hash": "52ea38d02ac4ae3e",729 "nbytes": 1638400,730 "offset": 54657024,731 "shape": [732 640,733 640734 ],735 "shard": "objects/b4e0145ce1f2ff18.fshard"736 },737 "blocks.7.attn.wo.weight": {738 "dtype": "f32",739 "hash": "945ecf9b46aeb099",740 "nbytes": 1638400,741 "offset": 57933824,742 "shape": [743 640,744 640745 ],746 "shard": "objects/b4e0145ce1f2ff18.fshard"747 },748 "blocks.7.attn.wq.weight": {749 "dtype": "f32",750 "hash": "bb4f532e47bbb588",751 "nbytes": 1638400,752 "offset": 53018624,753 "shape": [754 640,755 640756 ],757 "shard": "objects/b4e0145ce1f2ff18.fshard"758 },759 "blocks.7.attn.wv.weight": {760 "dtype": "f32",761 "hash": "39fc5bb55f990e08",762 "nbytes": 1638400,763 "offset": 56295424,764 "shape": [765 640,766 640767 ],768 "shard": "objects/b4e0145ce1f2ff18.fshard"769 },770 "blocks.7.mlp.w1.weight": {771 "dtype": "f32",772 "hash": "661591194873739a",773 "nbytes": 4423680,774 "offset": 59588608,775 "shape": [776 1728,777 640778 ],779 "shard": "objects/b4e0145ce1f2ff18.fshard"780 },781 "blocks.7.mlp.w2.weight": {782 "dtype": "f32",783 "hash": "c0693bdee8cb26f0",784 "nbytes": 4423680,785 "offset": 68435968,786 "shape": [787 640,788 1728789 ],790 "shard": "objects/b4e0145ce1f2ff18.fshard"791 },792 "blocks.7.mlp.w3.weight": {793 "dtype": "f32",794 "hash": "c46311e050bbc2e7",795 "nbytes": 4423680,796 "offset": 64012288,797 "shape": [798 1728,799 640800 ],801 "shard": "objects/b4e0145ce1f2ff18.fshard"802 },803 "blocks.7.norm1.weight": {804 "dtype": "f32",805 "hash": "31ddf245176231e8",806 "nbytes": 2560,807 "offset": 53002240,808 "shape": [809 640810 ],811 "shard": "objects/b4e0145ce1f2ff18.fshard"812 },813 "blocks.7.norm2.weight": {814 "dtype": "f32",815 "hash": "adb0020bde0202ae",816 "nbytes": 2560,817 "offset": 59572224,818 "shape": [819 640820 ],821 "shard": "objects/b4e0145ce1f2ff18.fshard"822 },823 "blocks.8.attn.wk.weight": {824 "dtype": "f32",825 "hash": "8b49975f1bcab4b1",826 "nbytes": 1638400,827 "offset": 74514432,828 "shape": [829 640,830 640831 ],832 "shard": "objects/b4e0145ce1f2ff18.fshard"833 },834 "blocks.8.attn.wo.weight": {835 "dtype": "f32",836 "hash": "d0fd8ea0a57aaf34",837 "nbytes": 1638400,838 "offset": 77791232,839 "shape": [840 640,841 640842 ],843 "shard": "objects/b4e0145ce1f2ff18.fshard"844 },845 "blocks.8.attn.wq.weight": {846 "dtype": "f32",847 "hash": "be6bf060be0ceb6a",848 "nbytes": 1638400,849 "offset": 72876032,850 "shape": [851 640,852 640853 ],854 "shard": "objects/b4e0145ce1f2ff18.fshard"855 },856 "blocks.8.attn.wv.weight": {857 "dtype": "f32",858 "hash": "876779888157dd34",859 "nbytes": 1638400,860 "offset": 76152832,861 "shape": [862 640,863 640864 ],865 "shard": "objects/b4e0145ce1f2ff18.fshard"866 },867 "blocks.8.mlp.w1.weight": {868 "dtype": "f32",869 "hash": "551d9ffdddc55001",870 "nbytes": 4423680,871 "offset": 79446016,872 "shape": [873 1728,874 640875 ],876 "shard": "objects/b4e0145ce1f2ff18.fshard"877 },878 "blocks.8.mlp.w2.weight": {879 "dtype": "f32",880 "hash": "710446ba331dbb33",881 "nbytes": 4423680,882 "offset": 88293376,883 "shape": [884 640,885 1728886 ],887 "shard": "objects/b4e0145ce1f2ff18.fshard"888 },889 "blocks.8.mlp.w3.weight": {890 "dtype": "f32",891 "hash": "75404c4f6c52ca7b",892 "nbytes": 4423680,893 "offset": 83869696,894 "shape": [895 1728,896 640897 ],898 "shard": "objects/b4e0145ce1f2ff18.fshard"899 },900 "blocks.8.norm1.weight": {901 "dtype": "f32",902 "hash": "a67c53d98e996f4d",903 "nbytes": 2560,904 "offset": 72859648,905 "shape": [906 640907 ],908 "shard": "objects/b4e0145ce1f2ff18.fshard"909 },910 "blocks.8.norm2.weight": {911 "dtype": "f32",912 "hash": "b7cc1cd235754b8d",913 "nbytes": 2560,914 "offset": 79429632,915 "shape": [916 640917 ],918 "shard": "objects/b4e0145ce1f2ff18.fshard"919 },920 "blocks.9.attn.wk.weight": {921 "dtype": "f32",922 "hash": "77ca114cacd59f92",923 "nbytes": 1638400,924 "offset": 94371840,925 "shape": [926 640,927 640928 ],929 "shard": "objects/b4e0145ce1f2ff18.fshard"930 },931 "blocks.9.attn.wo.weight": {932 "dtype": "f32",933 "hash": "2bd03b1679da1e59",934 "nbytes": 1638400,935 "offset": 97648640,936 "shape": [937 640,938 640939 ],940 "shard": "objects/b4e0145ce1f2ff18.fshard"941 },942 "blocks.9.attn.wq.weight": {943 "dtype": "f32",944 "hash": "c4ec432807c49aea",945 "nbytes": 1638400,946 "offset": 92733440,947 "shape": [948 640,949 640950 ],951 "shard": "objects/b4e0145ce1f2ff18.fshard"952 },953 "blocks.9.attn.wv.weight": {954 "dtype": "f32",955 "hash": "4f2bdf517e8aa817",956 "nbytes": 1638400,957 "offset": 96010240,958 "shape": [959 640,960 640961 ],962 "shard": "objects/b4e0145ce1f2ff18.fshard"963 },964 "blocks.9.mlp.w1.weight": {965 "dtype": "f32",966 "hash": "f65ccd0b671cebed",967 "nbytes": 4423680,968 "offset": 16384,969 "shape": [970 1728,971 640972 ],973 "shard": "objects/24a9ba4df0fd71ed.fshard"974 },975 "blocks.9.mlp.w2.weight": {976 "dtype": "f32",977 "hash": "14f2da31880d173e",978 "nbytes": 4423680,979 "offset": 8863744,980 "shape": [981 640,982 1728983 ],984 "shard": "objects/24a9ba4df0fd71ed.fshard"985 },986 "blocks.9.mlp.w3.weight": {987 "dtype": "f32",988 "hash": "08871f04b2cb1bb0",989 "nbytes": 4423680,990 "offset": 4440064,991 "shape": [992 1728,993 640994 ],995 "shard": "objects/24a9ba4df0fd71ed.fshard"996 },997 "blocks.9.norm1.weight": {998 "dtype": "f32",999 "hash": "ce6abe2627974147",1000 "nbytes": 2560,1001 "offset": 92717056,1002 "shape": [1003 6401004 ],1005 "shard": "objects/b4e0145ce1f2ff18.fshard"1006 },1007 "blocks.9.norm2.weight": {1008 "dtype": "f32",1009 "hash": "cee537f25da157b9",1010 "nbytes": 2560,1011 "offset": 99287040,1012 "shape": [1013 6401014 ],1015 "shard": "objects/b4e0145ce1f2ff18.fshard"1016 },1017 "final_norm.weight": {1018 "dtype": "f32",1019 "hash": "e14ca576c57ef871",1020 "nbytes": 2560,1021 "offset": 13287424,1022 "shape": [1023 6401024 ],1025 "shard": "objects/24a9ba4df0fd71ed.fshard"1026 },1027 "tok_emb.weight": {1028 "dtype": "f32",1029 "hash": "1d963329140f196a",1030 "nbytes": 10485760,1031 "offset": 16384,1032 "shape": [1033 4096,1034 6401035 ],1036 "shard": "objects/72626eb74479f8dd.fshard"1037 }1038 }1039}