seed: 1
model: configs/prisme_103m.yaml
model_overrides:
  readout_zero_init: false
  readout_init_std: 0.2
  ctrl_input: h
  ctrl_use_entropy: true
init:
  backbone_checkpoint: experiments/runs/PRETRAIN103M_RUN/checkpoint
  freeze_backbone: true
  auto_readout_scale: true
data:
  kind: tokens
  train_bin: data/processed/fineweb_edu_300m/train.bin
  val_bin: data/processed/fineweb_edu_300m/val.bin
budgets:
- 0
- 1
- 2
- 4
- 8
budget_probs:
- 0.2
- 0.2
- 0.2
- 0.2
- 0.2
budget_per: position
train:
  batch_size: 16
  seq_len: 512
  steps: 3000
  lr: 0.001
  warmup: 100
  weight_decay: 0.1
  grad_clip: 1.0
  log_every: 20
  eval_every: 500
  eval_batches: 10
  ckpt_every: 1000
final_exact_match: false
name: gr_pilot_103m
mode: global_recurrence
baseline_kwargs:
  ffn_w: 2560