SPB Git

spb/wp31_uqo

Public
1experiment:2  seed: 13  model: configs/prisme_103m.yaml4  model_overrides:5    readout_zero_init: false6    readout_init_std: 0.27    ctrl_input: h8    ctrl_use_entropy: true9  init:10    backbone_checkpoint: experiments/runs/pretrain103m_seed01_20261029232706/checkpoint11    freeze_backbone: true12    auto_readout_scale: true13  data:14    kind: tokens15    train_bin: data/processed/fineweb_edu_300m/train.bin16    val_bin: data/processed/fineweb_edu_300m/val.bin17  budgets:18  - 019  - 120  - 221  - 422  - 823  budget_probs:24  - 0.225  - 0.226  - 0.227  - 0.228  - 0.229  budget_per: position30  train:31    batch_size: 1632    seq_len: 51233    steps: 300034    lr: 0.00135    warmup: 10036    weight_decay: 0.137    grad_clip: 1.038    log_every: 2039    eval_every: 50040    eval_batches: 1041    ckpt_every: 100042  final_exact_match: false43  name: gr_pilot_103m44  mode: global_recurrence45  baseline_kwargs:46    ffn_w: 256047model:48  name: prisme_103m49  vocab_size: 3276850  d_model: 76851  n_layers: 1252  n_heads: 1253  n_kv_heads: 354  ffn_width: 204855  max_seq_len: 409656  rope_theta: 10000.057  norm_eps: 1.0e-0558  tie_embeddings: true59  init_std: 0.0260  use_workspace: true61  w: 25662  ffn_w: 76863  n_slots: 1664  n_editor_blocks: 265  n_editor_heads: 466  r_max: 867  eta_max: 0.2568  clip_c: 1.069  clip_eps: 1.0e-0670  readout_zero_init: false71  readout_init_std: 0.272  readout_scale: 1.073  controller_hidden: 074  ctrl_input: h75  ctrl_use_entropy: true76  lambda_cost: 0.077  kappa_risk: 0.078  dense_writes: false79resolved_train:80  batch_size: 1681  seq_len: 51282  steps: 300083  lr: 0.00184  warmup: 10085  weight_decay: 0.186  grad_clip: 1.087  log_every: 2088  eval_every: 50089  eval_batches: 1090  ckpt_every: 100091
Files · main
Menu

Theme