SPB Git

spb/wp31_uqo

Public
1experiment:2  seed: 13  model: configs/prisme_103m.yaml4  model_overrides:5    readout_zero_init: false6    readout_init_std: 0.27    ctrl_input: h8    ctrl_use_entropy: true9  init:10    backbone_checkpoint: experiments/runs/pretrain103m_seed01_20261029232706/checkpoint11    freeze_backbone: true12    auto_readout_scale: true13  data:14    kind: tokens15    train_bin: data/processed/fineweb_edu_300m/train.bin16    val_bin: data/processed/fineweb_edu_300m/val.bin17  budgets:18  - 019  - 120  - 221  - 422  - 823  budget_probs:24  - 0.225  - 0.226  - 0.227  - 0.228  - 0.229  budget_per: position30  train:31    batch_size: 1632    seq_len: 51233    steps: 300034    lr: 0.000335    warmup: 10036    weight_decay: 0.137    grad_clip: 1.038    log_every: 2039    eval_every: 50040    eval_batches: 1041    ckpt_every: 100042  final_exact_match: false43  name: editor_pilot_103m_lr3e444  mode: prisme45model:46  name: prisme_103m47  vocab_size: 3276848  d_model: 76849  n_layers: 1250  n_heads: 1251  n_kv_heads: 352  ffn_width: 204853  max_seq_len: 409654  rope_theta: 10000.055  norm_eps: 1.0e-0556  tie_embeddings: true57  init_std: 0.0258  use_workspace: true59  w: 25660  ffn_w: 76861  n_slots: 1662  n_editor_blocks: 263  n_editor_heads: 464  r_max: 865  eta_max: 0.2566  clip_c: 1.067  clip_eps: 1.0e-0668  readout_zero_init: false69  readout_init_std: 0.270  readout_scale: 1.071  controller_hidden: 072  ctrl_input: h73  ctrl_use_entropy: true74  lambda_cost: 0.075  kappa_risk: 0.076  dense_writes: false77resolved_train:78  batch_size: 1679  seq_len: 51280  steps: 300081  lr: 0.000382  warmup: 10083  weight_decay: 0.184  grad_clip: 1.085  log_every: 2086  eval_every: 50087  eval_batches: 1088  ckpt_every: 100089
Files · main
Menu

Theme