experiment:
  name: pretrain103m
  seed: 1
  model: configs/prisme_103m.yaml
  data:
    train_bin: data/processed/fineweb_edu_300m/train.bin
    val_bin: data/processed/fineweb_edu_300m/val.bin
  train: &id001
    batch_size: 32
    seq_len: 512
    steps: 6000
    lr: 0.0006
    warmup: 300
    weight_decay: 0.1
    grad_clip: 1.0
    log_every: 20
    eval_every: 500
    eval_batches: 20
    ckpt_every: 1000
model:
  name: prisme_103m
  vocab_size: 32768
  d_model: 768
  n_layers: 12
  n_heads: 12
  n_kv_heads: 3
  ffn_width: 2048
  max_seq_len: 4096
  rope_theta: 10000.0
  norm_eps: 1.0e-05
  tie_embeddings: true
  init_std: 0.02
  use_workspace: false
  w: 256
  ffn_w: 768
  n_slots: 16
  n_editor_blocks: 2
  n_editor_heads: 4
  r_max: 8
  eta_max: 0.25
  clip_c: 1.0
  clip_eps: 1.0e-06
  readout_zero_init: true
  controller_hidden: 0
  lambda_cost: 0.0
  kappa_risk: 0.0
  dense_writes: false
resolved_train: *id001