1{2 "source": "HuggingFaceFW/fineweb-edu sample-10BT (streaming, canonical shard order)",3 "license": "ODC-By 1.0 (https://opendatacommons.org/licenses/by/1-0/)",4 "prepared_at": "2026-10-29T23:19:53-0400",5 "args": {6 "out": "data/processed/fineweb_edu_300m",7 "target_train_tokens": 300000000,8 "vocab_size": 32768,9 "tokenizer_docs": 60000,10 "val_pct": 0.01,11 "test_pct": 0.01,12 "max_doc_chars": 4000013 },14 "docs_streamed": 291647,15 "docs": {16 "train": 285604,17 "val": 2957,18 "test": 297319 },20 "chars": {21 "train": 1290001347,22 "val": 12988848,23 "test": 1293303224 },25 "splits": {26 "train": {27 "tokens": 289959006,28 "docs": 285604,29 "sha256": "e931e83c4ba6e62a2125e50a5390e8df78913dbed31af805d8af17fafad95123",30 "locked": false31 },32 "val": {33 "tokens": 2913824,34 "docs": 2957,35 "sha256": "0565e3f4a4028b241f40ef07f20bc68b24e45d99e147860be066e3667d1532be",36 "locked": false37 },38 "test": {39 "tokens": 2883436,40 "docs": 2973,41 "sha256": "1ce9ca3ab6d659fe08042844fc0589951e09386ee79d2ef76504ffaaebb0947b",42 "locked": true43 }44 },45 "tokenizer_sha256": "5a78df618767945743623d5ff238d035839d02388b4c53c548a437b3d8a206ee",46 "eot_id": 0,47 "dtype": "uint16"48}