{
  "source": "HuggingFaceFW/fineweb-edu sample-10BT (streaming, canonical shard order)",
  "license": "ODC-By 1.0 (https://opendatacommons.org/licenses/by/1-0/)",
  "prepared_at": "2026-10-29T23:19:53-0400",
  "args": {
    "out": "data/processed/fineweb_edu_300m",
    "target_train_tokens": 300000000,
    "vocab_size": 32768,
    "tokenizer_docs": 60000,
    "val_pct": 0.01,
    "test_pct": 0.01,
    "max_doc_chars": 40000
  },
  "docs_streamed": 291647,
  "docs": {
    "train": 285604,
    "val": 2957,
    "test": 2973
  },
  "chars": {
    "train": 1290001347,
    "val": 12988848,
    "test": 12933032
  },
  "splits": {
    "train": {
      "tokens": 289959006,
      "docs": 285604,
      "sha256": "e931e83c4ba6e62a2125e50a5390e8df78913dbed31af805d8af17fafad95123",
      "locked": false
    },
    "val": {
      "tokens": 2913824,
      "docs": 2957,
      "sha256": "0565e3f4a4028b241f40ef07f20bc68b24e45d99e147860be066e3667d1532be",
      "locked": false
    },
    "test": {
      "tokens": 2883436,
      "docs": 2973,
      "sha256": "1ce9ca3ab6d659fe08042844fc0589951e09386ee79d2ef76504ffaaebb0947b",
      "locked": true
    }
  },
  "tokenizer_sha256": "5a78df618767945743623d5ff238d035839d02388b4c53c548a437b3d8a206ee",
  "eot_id": 0,
  "dtype": "uint16"
}