SPB Git

spb/wp31_uqo

Public
1experiment:2  name: mini_backbone3  seed: 14  mode: backbone_only5  model:6    name: mini7    vocab_size: 1288    d_model: 969    n_layers: 210    n_heads: 411    n_kv_heads: 212    ffn_width: 25613    max_seq_len: 32014    w: 3215    ffn_w: 9616    n_slots: 1617    n_editor_heads: 218    r_max: 819    readout_zero_init: false20    readout_init_std: 0.221    ctrl_input: h22    ctrl_use_entropy: true23  init:24    freeze_backbone: false25  data:26    kind: synthetic27    tokenizer: char28    splits_dir: data/splits/synthetic_v129    loss_span: all30    eval_splits:31    - dev_iid32  train:33    batch_size: 1634    seq_len: 32035    steps: 15036    lr: 0.00137    warmup: 1038    weight_decay: 0.139    grad_clip: 1.040    log_every: 5041    eval_every: 15042    eval_batches: 243    ckpt_every: 15044  em_max_items: 3245model:46  name: mini47  vocab_size: 12848  d_model: 9649  n_layers: 250  n_heads: 451  n_kv_heads: 252  ffn_width: 25653  max_seq_len: 32054  rope_theta: 10000.055  norm_eps: 1.0e-0556  tie_embeddings: true57  init_std: 0.0258  use_workspace: false59  w: 3260  ffn_w: 9661  n_slots: 1662  n_editor_blocks: 263  n_editor_heads: 264  r_max: 865  eta_max: 0.2566  clip_c: 1.067  clip_eps: 1.0e-0668  readout_zero_init: false69  readout_init_std: 0.270  readout_scale: 1.071  controller_hidden: 072  ctrl_input: h73  ctrl_use_entropy: true74  lambda_cost: 0.075  kappa_risk: 0.076  dense_writes: false77resolved_train:78  batch_size: 1679  seq_len: 32080  steps: 15081  lr: 0.00182  warmup: 1083  weight_decay: 0.184  grad_clip: 1.085  log_every: 5086  eval_every: 15087  eval_batches: 288  ckpt_every: 15089
Files · main
Menu

Theme