Repository Public Research preview
1{2 "run_id": "pilot_tier0_opseq_451ef5087182",3 "config_hash": "451ef5087182",4 "device": "mps",5 "chance_level": {6 "target_iid": 0.105,7 "control_iid": 0.129,8 "comp_heldout": 0.11,9 "len_ood": 0.1410 },11 "seeds": {12 "0": {13 "donor": {14 "n_params": 800000,15 "train_seconds": 45.07666206359863,16 "final_dev": {17 "step": 3000,18 "loss": 1.4056669606077321e-08,19 "dev_acc": 1.0,20 "dev_trace_acc": 1.0,21 "lr": 0.0,22 "elapsed_s": 44.9931719303131123 },24 "eval": {25 "target_iid": {26 "split": "target_iid",27 "split_hash": "f1a1ef205c8f35db",28 "n": 1000,29 "accuracy": 1.0,30 "trace_accuracy": 1.0,31 "ci95": {32 "estimate": 1.0,33 "low": 1.0,34 "high": 1.0,35 "level": 0.95,36 "n_items": 1000,37 "n_clusters": 944,38 "n_boot": 200039 },40 "omega_step_acc": {41 "estimate": 1.0,42 "low": 1.0,43 "high": 1.0,44 "level": 0.95,45 "n_items": 1000,46 "n_clusters": 944,47 "n_boot": 200048 },49 "chance": 0.10550 },51 "control_iid": {52 "split": "control_iid",53 "split_hash": "fa7c2327072b7d56",54 "n": 1000,55 "accuracy": 1.0,56 "trace_accuracy": 1.0,57 "ci95": {58 "estimate": 1.0,59 "low": 1.0,60 "high": 1.0,61 "level": 0.95,62 "n_items": 1000,63 "n_clusters": 970,64 "n_boot": 200065 },66 "omega_step_acc": null,67 "chance": 0.12968 },69 "comp_heldout": {70 "split": "comp_heldout",71 "split_hash": "b1c6aebefbf3a5d5",72 "n": 1000,73 "accuracy": 1.0,74 "trace_accuracy": 1.0,75 "ci95": {76 "estimate": 1.0,77 "low": 1.0,78 "high": 1.0,79 "level": 0.95,80 "n_items": 1000,81 "n_clusters": 681,82 "n_boot": 200083 },84 "omega_step_acc": {85 "estimate": 1.0,86 "low": 1.0,87 "high": 1.0,88 "level": 0.95,89 "n_items": 592,90 "n_clusters": 420,91 "n_boot": 200092 },93 "chance": 0.1194 },95 "len_ood": {96 "split": "len_ood",97 "split_hash": "7ecb075ac45796ac",98 "n": 500,99 "accuracy": 0.076,100 "trace_accuracy": 0.0,101 "ci95": {102 "estimate": 0.076,103 "low": 0.054,104 "high": 0.1,105 "level": 0.95,106 "n_items": 500,107 "n_clusters": 500,108 "n_boot": 2000109 },110 "omega_step_acc": {111 "estimate": 0.09830508474576272,112 "low": 0.06440677966101695,113 "high": 0.13220338983050847,114 "level": 0.95,115 "n_items": 295,116 "n_clusters": 295,117 "n_boot": 2000118 },119 "chance": 0.14120 }121 }122 },123 "recipient": {124 "n_params": 800000,125 "train_seconds": 44.30060911178589,126 "final_dev": {127 "step": 3000,128 "loss": 6.440794120265991e-09,129 "dev_acc": 1.0,130 "dev_trace_acc": 1.0,131 "lr": 0.0,132 "elapsed_s": 44.22483992576599133 },134 "eval": {135 "target_iid": {136 "split": "target_iid",137 "split_hash": "f1a1ef205c8f35db",138 "n": 1000,139 "accuracy": 0.153,140 "trace_accuracy": 0.11,141 "ci95": {142 "estimate": 0.153,143 "low": 0.13021422874046046,144 "high": 0.17482830290981383,145 "level": 0.95,146 "n_items": 1000,147 "n_clusters": 944,148 "n_boot": 2000149 },150 "omega_step_acc": {151 "estimate": 0.142,152 "low": 0.12102406103286387,153 "high": 0.16371086200231927,154 "level": 0.95,155 "n_items": 1000,156 "n_clusters": 944,157 "n_boot": 2000158 },159 "chance": 0.105160 },161 "control_iid": {162 "split": "control_iid",163 "split_hash": "fa7c2327072b7d56",164 "n": 1000,165 "accuracy": 1.0,166 "trace_accuracy": 1.0,167 "ci95": {168 "estimate": 1.0,169 "low": 1.0,170 "high": 1.0,171 "level": 0.95,172 "n_items": 1000,173 "n_clusters": 970,174 "n_boot": 2000175 },176 "omega_step_acc": null,177 "chance": 0.129178 },179 "comp_heldout": {180 "split": "comp_heldout",181 "split_hash": "b1c6aebefbf3a5d5",182 "n": 1000,183 "accuracy": 0.487,184 "trace_accuracy": 0.466,185 "ci95": {186 "estimate": 0.487,187 "low": 0.4403249441330549,188 "high": 0.5318091451292246,189 "level": 0.95,190 "n_items": 1000,191 "n_clusters": 681,192 "n_boot": 2000193 },194 "omega_step_acc": {195 "estimate": 0.13006756756756757,196 "low": 0.10363636363636364,197 "high": 0.15476609657947685,198 "level": 0.95,199 "n_items": 592,200 "n_clusters": 420,201 "n_boot": 2000202 },203 "chance": 0.11204 },205 "len_ood": {206 "split": "len_ood",207 "split_hash": "7ecb075ac45796ac",208 "n": 500,209 "accuracy": 0.078,210 "trace_accuracy": 0.0,211 "ci95": {212 "estimate": 0.078,213 "low": 0.054,214 "high": 0.102,215 "level": 0.95,216 "n_items": 500,217 "n_clusters": 500,218 "n_boot": 2000219 },220 "omega_step_acc": {221 "estimate": 0.11186440677966102,222 "low": 0.07796610169491526,223 "high": 0.14576271186440679,224 "level": 0.95,225 "n_items": 295,226 "n_clusters": 295,227 "n_boot": 2000228 },229 "chance": 0.14230 }231 }232 },233 "gap": {234 "target_iid": {235 "estimate": 0.847,236 "low": 0.8251716970901862,237 "high": 0.8697857712595395,238 "level": 0.95,239 "n_items": 1000,240 "n_clusters": 944,241 "n_boot": 2000242 },243 "control_iid": {244 "estimate": 0.0,245 "low": 0.0,246 "high": 0.0,247 "level": 0.95,248 "n_items": 1000,249 "n_clusters": 970,250 "n_boot": 2000251 },252 "comp_heldout": {253 "estimate": 0.513,254 "low": 0.46819085487077533,255 "high": 0.5596750558669451,256 "level": 0.95,257 "n_items": 1000,258 "n_clusters": 681,259 "n_boot": 2000260 },261 "len_ood": {262 "estimate": -0.002,263 "low": -0.034,264 "high": 0.032,265 "level": 0.95,266 "n_items": 500,267 "n_clusters": 500,268 "n_boot": 2000269 }270 }271 },272 "1": {273 "donor": {274 "n_params": 800000,275 "train_seconds": 44.552977085113525,276 "final_dev": {277 "step": 3000,278 "loss": 2.2240537189688325e-10,279 "dev_acc": 1.0,280 "dev_trace_acc": 1.0,281 "lr": 0.0,282 "elapsed_s": 44.468523025512695283 },284 "eval": {285 "target_iid": {286 "split": "target_iid",287 "split_hash": "f1a1ef205c8f35db",288 "n": 1000,289 "accuracy": 1.0,290 "trace_accuracy": 1.0,291 "ci95": {292 "estimate": 1.0,293 "low": 1.0,294 "high": 1.0,295 "level": 0.95,296 "n_items": 1000,297 "n_clusters": 944,298 "n_boot": 2000299 },300 "omega_step_acc": {301 "estimate": 1.0,302 "low": 1.0,303 "high": 1.0,304 "level": 0.95,305 "n_items": 1000,306 "n_clusters": 944,307 "n_boot": 2000308 },309 "chance": 0.105310 },311 "control_iid": {312 "split": "control_iid",313 "split_hash": "fa7c2327072b7d56",314 "n": 1000,315 "accuracy": 1.0,316 "trace_accuracy": 1.0,317 "ci95": {318 "estimate": 1.0,319 "low": 1.0,320 "high": 1.0,321 "level": 0.95,322 "n_items": 1000,323 "n_clusters": 970,324 "n_boot": 2000325 },326 "omega_step_acc": null,327 "chance": 0.129328 },329 "comp_heldout": {330 "split": "comp_heldout",331 "split_hash": "b1c6aebefbf3a5d5",332 "n": 1000,333 "accuracy": 1.0,334 "trace_accuracy": 1.0,335 "ci95": {336 "estimate": 1.0,337 "low": 1.0,338 "high": 1.0,339 "level": 0.95,340 "n_items": 1000,341 "n_clusters": 681,342 "n_boot": 2000343 },344 "omega_step_acc": {345 "estimate": 1.0,346 "low": 1.0,347 "high": 1.0,348 "level": 0.95,349 "n_items": 592,350 "n_clusters": 420,351 "n_boot": 2000352 },353 "chance": 0.11354 },355 "len_ood": {356 "split": "len_ood",357 "split_hash": "7ecb075ac45796ac",358 "n": 500,359 "accuracy": 0.076,360 "trace_accuracy": 0.0,361 "ci95": {362 "estimate": 0.076,363 "low": 0.054,364 "high": 0.1,365 "level": 0.95,366 "n_items": 500,367 "n_clusters": 500,368 "n_boot": 2000369 },370 "omega_step_acc": {371 "estimate": 0.16271186440677965,372 "low": 0.12203389830508475,373 "high": 0.20677966101694914,374 "level": 0.95,375 "n_items": 295,376 "n_clusters": 295,377 "n_boot": 2000378 },379 "chance": 0.14380 }381 }382 },383 "recipient": {384 "n_params": 800000,385 "train_seconds": 44.89582920074463,386 "final_dev": {387 "step": 3000,388 "loss": 2.1306395536768719e-10,389 "dev_acc": 1.0,390 "dev_trace_acc": 1.0,391 "lr": 0.0,392 "elapsed_s": 44.8197238445282393 },394 "eval": {395 "target_iid": {396 "split": "target_iid",397 "split_hash": "f1a1ef205c8f35db",398 "n": 1000,399 "accuracy": 0.064,400 "trace_accuracy": 0.024,401 "ci95": {402 "estimate": 0.064,403 "low": 0.04909696770919216,404 "high": 0.07923771313941826,405 "level": 0.95,406 "n_items": 1000,407 "n_clusters": 944,408 "n_boot": 2000409 },410 "omega_step_acc": {411 "estimate": 0.026,412 "low": 0.01693227091633466,413 "high": 0.03625377643504532,414 "level": 0.95,415 "n_items": 1000,416 "n_clusters": 944,417 "n_boot": 2000418 },419 "chance": 0.105420 },421 "control_iid": {422 "split": "control_iid",423 "split_hash": "fa7c2327072b7d56",424 "n": 1000,425 "accuracy": 1.0,426 "trace_accuracy": 1.0,427 "ci95": {428 "estimate": 1.0,429 "low": 1.0,430 "high": 1.0,431 "level": 0.95,432 "n_items": 1000,433 "n_clusters": 970,434 "n_boot": 2000435 },436 "omega_step_acc": null,437 "chance": 0.129438 },439 "comp_heldout": {440 "split": "comp_heldout",441 "split_hash": "b1c6aebefbf3a5d5",442 "n": 1000,443 "accuracy": 0.435,444 "trace_accuracy": 0.421,445 "ci95": {446 "estimate": 0.435,447 "low": 0.38675184162062615,448 "high": 0.48358677815112916,449 "level": 0.95,450 "n_items": 1000,451 "n_clusters": 681,452 "n_boot": 2000453 },454 "omega_step_acc": {455 "estimate": 0.02364864864864865,456 "low": 0.01279649114613219,457 "high": 0.035415924087713664,458 "level": 0.95,459 "n_items": 592,460 "n_clusters": 420,461 "n_boot": 2000462 },463 "chance": 0.11464 },465 "len_ood": {466 "split": "len_ood",467 "split_hash": "7ecb075ac45796ac",468 "n": 500,469 "accuracy": 0.086,470 "trace_accuracy": 0.0,471 "ci95": {472 "estimate": 0.086,473 "low": 0.062,474 "high": 0.11004999999999973,475 "level": 0.95,476 "n_items": 500,477 "n_clusters": 500,478 "n_boot": 2000479 },480 "omega_step_acc": {481 "estimate": 0.0711864406779661,482 "low": 0.04406779661016949,483 "high": 0.1016949152542373,484 "level": 0.95,485 "n_items": 295,486 "n_clusters": 295,487 "n_boot": 2000488 },489 "chance": 0.14490 }491 }492 },493 "gap": {494 "target_iid": {495 "estimate": 0.936,496 "low": 0.9207622868605817,497 "high": 0.9509030322908079,498 "level": 0.95,499 "n_items": 1000,500 "n_clusters": 944,501 "n_boot": 2000502 },503 "control_iid": {504 "estimate": 0.0,505 "low": 0.0,506 "high": 0.0,507 "level": 0.95,508 "n_items": 1000,509 "n_clusters": 970,510 "n_boot": 2000511 },512 "comp_heldout": {513 "estimate": 0.565,514 "low": 0.5164132218488708,515 "high": 0.6132481583793739,516 "level": 0.95,517 "n_items": 1000,518 "n_clusters": 681,519 "n_boot": 2000520 },521 "len_ood": {522 "estimate": -0.01,523 "low": -0.044,524 "high": 0.02,525 "level": 0.95,526 "n_items": 500,527 "n_clusters": 500,528 "n_boot": 2000529 }530 }531 },532 "2": {533 "donor": {534 "n_params": 800000,535 "train_seconds": 44.78438401222229,536 "final_dev": {537 "step": 3000,538 "loss": 0.0,539 "dev_acc": 1.0,540 "dev_trace_acc": 1.0,541 "lr": 0.0,542 "elapsed_s": 44.687108755111694543 },544 "eval": {545 "target_iid": {546 "split": "target_iid",547 "split_hash": "f1a1ef205c8f35db",548 "n": 1000,549 "accuracy": 1.0,550 "trace_accuracy": 1.0,551 "ci95": {552 "estimate": 1.0,553 "low": 1.0,554 "high": 1.0,555 "level": 0.95,556 "n_items": 1000,557 "n_clusters": 944,558 "n_boot": 2000559 },560 "omega_step_acc": {561 "estimate": 1.0,562 "low": 1.0,563 "high": 1.0,564 "level": 0.95,565 "n_items": 1000,566 "n_clusters": 944,567 "n_boot": 2000568 },569 "chance": 0.105570 },571 "control_iid": {572 "split": "control_iid",573 "split_hash": "fa7c2327072b7d56",574 "n": 1000,575 "accuracy": 1.0,576 "trace_accuracy": 1.0,577 "ci95": {578 "estimate": 1.0,579 "low": 1.0,580 "high": 1.0,581 "level": 0.95,582 "n_items": 1000,583 "n_clusters": 970,584 "n_boot": 2000585 },586 "omega_step_acc": null,587 "chance": 0.129588 },589 "comp_heldout": {590 "split": "comp_heldout",591 "split_hash": "b1c6aebefbf3a5d5",592 "n": 1000,593 "accuracy": 1.0,594 "trace_accuracy": 1.0,595 "ci95": {596 "estimate": 1.0,597 "low": 1.0,598 "high": 1.0,599 "level": 0.95,600 "n_items": 1000,601 "n_clusters": 681,602 "n_boot": 2000603 },604 "omega_step_acc": {605 "estimate": 1.0,606 "low": 1.0,607 "high": 1.0,608 "level": 0.95,609 "n_items": 592,610 "n_clusters": 420,611 "n_boot": 2000612 },613 "chance": 0.11614 },615 "len_ood": {616 "split": "len_ood",617 "split_hash": "7ecb075ac45796ac",618 "n": 500,619 "accuracy": 0.106,620 "trace_accuracy": 0.0,621 "ci95": {622 "estimate": 0.106,623 "low": 0.08,624 "high": 0.134,625 "level": 0.95,626 "n_items": 500,627 "n_clusters": 500,628 "n_boot": 2000629 },630 "omega_step_acc": {631 "estimate": 0.15593220338983052,632 "low": 0.1152542372881356,633 "high": 0.2,634 "level": 0.95,635 "n_items": 295,636 "n_clusters": 295,637 "n_boot": 2000638 },639 "chance": 0.14640 }641 }642 },643 "recipient": {644 "n_params": 800000,645 "train_seconds": 43.506938219070435,646 "final_dev": {647 "step": 3000,648 "loss": 7.384645672736667e-10,649 "dev_acc": 1.0,650 "dev_trace_acc": 1.0,651 "lr": 0.0,652 "elapsed_s": 43.429972887039185653 },654 "eval": {655 "target_iid": {656 "split": "target_iid",657 "split_hash": "f1a1ef205c8f35db",658 "n": 1000,659 "accuracy": 0.059,660 "trace_accuracy": 0.013,661 "ci95": {662 "estimate": 0.059,663 "low": 0.04481960218826188,664 "high": 0.07522622322412784,665 "level": 0.95,666 "n_items": 1000,667 "n_clusters": 944,668 "n_boot": 2000669 },670 "omega_step_acc": {671 "estimate": 0.017,672 "low": 0.009910312626519584,673 "high": 0.025768726304529074,674 "level": 0.95,675 "n_items": 1000,676 "n_clusters": 944,677 "n_boot": 2000678 },679 "chance": 0.105680 },681 "control_iid": {682 "split": "control_iid",683 "split_hash": "fa7c2327072b7d56",684 "n": 1000,685 "accuracy": 1.0,686 "trace_accuracy": 1.0,687 "ci95": {688 "estimate": 1.0,689 "low": 1.0,690 "high": 1.0,691 "level": 0.95,692 "n_items": 1000,693 "n_clusters": 970,694 "n_boot": 2000695 },696 "omega_step_acc": null,697 "chance": 0.129698 },699 "comp_heldout": {700 "split": "comp_heldout",701 "split_hash": "b1c6aebefbf3a5d5",702 "n": 1000,703 "accuracy": 0.429,704 "trace_accuracy": 0.417,705 "ci95": {706 "estimate": 0.429,707 "low": 0.3773348057922526,708 "high": 0.47867104972724117,709 "level": 0.95,710 "n_items": 1000,711 "n_clusters": 681,712 "n_boot": 2000713 },714 "omega_step_acc": {715 "estimate": 0.02195945945945946,716 "low": 0.0105074216501537,717 "high": 0.03460972052542356,718 "level": 0.95,719 "n_items": 592,720 "n_clusters": 420,721 "n_boot": 2000722 },723 "chance": 0.11724 },725 "len_ood": {726 "split": "len_ood",727 "split_hash": "7ecb075ac45796ac",728 "n": 500,729 "accuracy": 0.108,730 "trace_accuracy": 0.0,731 "ci95": {732 "estimate": 0.108,733 "low": 0.082,734 "high": 0.136,735 "level": 0.95,736 "n_items": 500,737 "n_clusters": 500,738 "n_boot": 2000739 },740 "omega_step_acc": {741 "estimate": 0.08135593220338982,742 "low": 0.05084745762711865,743 "high": 0.11186440677966102,744 "level": 0.95,745 "n_items": 295,746 "n_clusters": 295,747 "n_boot": 2000748 },749 "chance": 0.14750 }751 }752 },753 "gap": {754 "target_iid": {755 "estimate": 0.941,756 "low": 0.9247737767758721,757 "high": 0.9551803978117381,758 "level": 0.95,759 "n_items": 1000,760 "n_clusters": 944,761 "n_boot": 2000762 },763 "control_iid": {764 "estimate": 0.0,765 "low": 0.0,766 "high": 0.0,767 "level": 0.95,768 "n_items": 1000,769 "n_clusters": 970,770 "n_boot": 2000771 },772 "comp_heldout": {773 "estimate": 0.571,774 "low": 0.5213289502727588,775 "high": 0.6226651942077474,776 "level": 0.95,777 "n_items": 1000,778 "n_clusters": 681,779 "n_boot": 2000780 },781 "len_ood": {782 "estimate": -0.002,783 "low": -0.038,784 "high": 0.034,785 "level": 0.95,786 "n_items": 500,787 "n_clusters": 500,788 "n_boot": 2000789 }790 }791 }792 },793 "noop": {794 "donor_s0": {795 "deterministic_repeat": true,796 "capture_all_bitwise_identity": true,797 "disabled_modifying_hook_bitwise_identity": true,798 "enabled_zero_hook_changes_output": true,799 "n_hook_points": 26,800 "n_items": 256801 },802 "recipient_s0": {803 "deterministic_repeat": true,804 "capture_all_bitwise_identity": true,805 "disabled_modifying_hook_bitwise_identity": true,806 "enabled_zero_hook_changes_output": true,807 "n_hook_points": 26,808 "n_items": 256809 },810 "donor_s1": {811 "deterministic_repeat": true,812 "capture_all_bitwise_identity": true,813 "disabled_modifying_hook_bitwise_identity": true,814 "enabled_zero_hook_changes_output": true,815 "n_hook_points": 26,816 "n_items": 256817 },818 "recipient_s1": {819 "deterministic_repeat": true,820 "capture_all_bitwise_identity": true,821 "disabled_modifying_hook_bitwise_identity": true,822 "enabled_zero_hook_changes_output": true,823 "n_hook_points": 26,824 "n_items": 256825 },826 "donor_s2": {827 "deterministic_repeat": true,828 "capture_all_bitwise_identity": true,829 "disabled_modifying_hook_bitwise_identity": true,830 "enabled_zero_hook_changes_output": true,831 "n_hook_points": 26,832 "n_items": 256833 },834 "recipient_s2": {835 "deterministic_repeat": true,836 "capture_all_bitwise_identity": true,837 "disabled_modifying_hook_bitwise_identity": true,838 "enabled_zero_hook_changes_output": true,839 "n_hook_points": 26,840 "n_items": 256841 }842 },843 "localization": {844 "n_items": 490,845 "read_position": 6,846 "positions": [847 "<bos>",848 "v0",849 "op1",850 "\u03a9",851 "op3",852 "=",853 "r1"854 ],855 "families": {856 "resid_post": {857 "s0": {858 "flip_rate": [859 [860 0.0,861 0.0,862 0.0,863 0.0,864 0.0,865 0.0,866 1.0867 ],868 [869 0.0,870 0.0,871 0.0,872 0.0,873 0.0,874 0.0,875 1.0876 ],877 [878 0.0,879 0.0,880 0.0,881 0.0,882 0.0,883 0.0,884 1.0885 ],886 [887 0.0,888 0.0,889 0.0,890 0.0,891 0.0,892 0.0,893 1.0894 ]895 ],896 "logit_recovery": [897 [898 0.0,899 0.0,900 0.0,901 -0.0029584874864667654,902 0.0002881789696402848,903 6.067532467568526e-06,904 0.9983479380607605905 ],906 [907 0.0,908 0.0,909 0.0,910 -0.000898381054867059,911 0.00018582800112199038,912 -7.255208038259298e-05,913 0.9993993639945984914 ],915 [916 0.0,917 0.0,918 0.0,919 7.875546725699678e-05,920 6.0847491113236174e-05,921 -9.433332888875157e-05,922 0.9998680949211121923 ],924 [925 0.0,926 0.0,927 0.0,928 0.0,929 0.0,930 0.0,931 1.0932 ]933 ]934 },935 "s1": {936 "flip_rate": [937 [938 0.0,939 0.0,940 0.0,941 0.0,942 0.0,943 0.0,944 1.0945 ],946 [947 0.0,948 0.0,949 0.0,950 0.0,951 0.0,952 0.0,953 1.0954 ],955 [956 0.0,957 0.0,958 0.0,959 0.0,960 0.0,961 0.0,962 1.0963 ],964 [965 0.0,966 0.0,967 0.0,968 0.0,969 0.0,970 0.0,971 1.0972 ]973 ],974 "logit_recovery": [975 [976 0.0,977 0.0,978 0.0,979 0.0004672935465350747,980 0.00017617014236748219,981 -1.2608468047403676e-08,982 0.9994205236434937983 ],984 [985 0.0,986 0.0,987 0.0,988 0.00011902503320015967,989 8.649702795082703e-05,990 4.641553630335693e-07,991 0.9994013905525208992 ],993 [994 0.0,995 0.0,996 0.0,997 3.435093822190538e-05,998 2.089686404360691e-06,999 1.4695291611133143e-05,1000 1.00023257732391361001 ],1002 [1003 0.0,1004 0.0,1005 0.0,1006 0.0,1007 0.0,1008 0.0,1009 1.01010 ]1011 ]1012 },1013 "s2": {1014 "flip_rate": [1015 [1016 0.0,1017 0.0,1018 0.0,1019 0.0,1020 0.0,1021 0.0,1022 1.01023 ],1024 [1025 0.0,1026 0.0,1027 0.0,1028 0.0,1029 0.0,1030 0.0,1031 1.01032 ],1033 [1034 0.0,1035 0.0,1036 0.0,1037 0.0,1038 0.0,1039 0.0,1040 1.01041 ],1042 [1043 0.0,1044 0.0,1045 0.0,1046 0.0,1047 0.0,1048 0.0,1049 1.01050 ]1051 ],1052 "logit_recovery": [1053 [1054 0.0,1055 0.0,1056 0.0,1057 0.00033169190282933414,1058 -4.887258546659723e-05,1059 1.5617764574926696e-06,1060 1.00012850761413571061 ],1062 [1063 0.0,1064 0.0,1065 0.0,1066 0.0004198819224257022,1067 -2.6704077754402533e-05,1068 1.01509458545479e-05,1069 0.99986666440963751070 ],1071 [1072 0.0,1073 0.0,1074 0.0,1075 8.019314554985613e-05,1076 1.6664354518525215e-07,1077 -1.2479076758609153e-06,1078 1.00024557113647461079 ],1080 [1081 0.0,1082 0.0,1083 0.0,1084 0.0,1085 0.0,1086 0.0,1087 1.01088 ]1089 ]1090 },1091 "mean_flip_rate": [1092 [1093 0.0,1094 0.0,1095 0.0,1096 0.0,1097 0.0,1098 0.0,1099 1.01100 ],1101 [1102 0.0,1103 0.0,1104 0.0,1105 0.0,1106 0.0,1107 0.0,1108 1.01109 ],1110 [1111 0.0,1112 0.0,1113 0.0,1114 0.0,1115 0.0,1116 0.0,1117 1.01118 ],1119 [1120 0.0,1121 0.0,1122 0.0,1123 0.0,1124 0.0,1125 0.0,1126 1.01127 ]1128 ]1129 },1130 "attn_out": {1131 "s0": {1132 "flip_rate": [1133 [1134 0.0,1135 0.0,1136 0.0,1137 0.0,1138 0.0,1139 0.0,1140 1.01141 ],1142 [1143 0.0,1144 0.0,1145 0.0,1146 0.0,1147 0.0,1148 0.0,1149 0.01150 ],1151 [1152 0.0,1153 0.0,1154 0.0,1155 0.0,1156 0.0,1157 0.0,1158 0.01159 ],1160 [1161 0.0,1162 0.0,1163 0.0,1164 0.0,1165 0.0,1166 0.0,1167 0.01168 ]1169 ],1170 "logit_recovery": [1171 [1172 0.0,1173 0.0,1174 0.0,1175 -0.0025477122981101274,1176 0.0002881789696402848,1177 6.067532467568526e-06,1178 0.99834793806076051179 ],1180 [1181 0.0,1182 0.0,1183 0.0,1184 -0.00013427382509689778,1185 -3.3803276892285794e-05,1186 -8.034344500629231e-05,1187 -0.00312549876980483531188 ],1189 [1190 0.0,1191 0.0,1192 0.0,1193 -2.6091085601365194e-05,1194 1.4899030247761402e-05,1195 -4.782400719705038e-05,1196 -0.0055566797964274881197 ],1198 [1199 0.0,1200 0.0,1201 0.0,1202 0.0,1203 0.0,1204 0.0,1205 -0.006832068320363761206 ]1207 ]1208 },1209 "s1": {1210 "flip_rate": [1211 [1212 0.0,1213 0.0,1214 0.0,1215 0.0,1216 0.0,1217 0.0,1218 1.01219 ],1220 [1221 0.0,1222 0.0,1223 0.0,1224 0.0,1225 0.0,1226 0.0,1227 0.01228 ],1229 [1230 0.0,1231 0.0,1232 0.0,1233 0.0,1234 0.0,1235 0.0,1236 0.01237 ],1238 [1239 0.0,1240 0.0,1241 0.0,1242 0.0,1243 0.0,1244 0.0,1245 0.01246 ]1247 ],1248 "logit_recovery": [1249 [1250 0.0,1251 0.0,1252 0.0,1253 0.000488389516249299,1254 0.00017617014236748219,1255 -1.2608468047403676e-08,1256 0.99942052364349371257 ],1258 [1259 0.0,1260 0.0,1261 0.0,1262 7.775469566695392e-05,1263 3.9568527427036315e-05,1264 5.730797170144797e-07,1265 0.00323216826654970651266 ],1267 [1268 0.0,1269 0.0,1270 0.0,1271 0.0002074985095532611,1272 2.0777781173819676e-05,1273 1.012419852486346e-05,1274 0.000236626059631817041275 ],1276 [1277 0.0,1278 0.0,1279 0.0,1280 0.0,1281 0.0,1282 0.0,1283 -0.0052552465349435811284 ]1285 ]1286 },1287 "s2": {1288 "flip_rate": [1289 [1290 0.0,1291 0.0,1292 0.0,1293 0.0,1294 0.0,1295 0.0,1296 1.01297 ],1298 [1299 0.0,1300 0.0,1301 0.0,1302 0.0,1303 0.0,1304 0.0,1305 0.01306 ],1307 [1308 0.0,1309 0.0,1310 0.0,1311 0.0,1312 0.0,1313 0.0,1314 0.01315 ],1316 [1317 0.0,1318 0.0,1319 0.0,1320 0.0,1321 0.0,1322 0.0,1323 0.01324 ]1325 ],1326 "logit_recovery": [1327 [1328 0.0,1329 0.0,1330 0.0,1331 0.00030576580320484936,1332 -4.887258546659723e-05,1333 1.5617764574926696e-06,1334 1.00012850761413571335 ],1336 [1337 0.0,1338 0.0,1339 0.0,1340 3.7193756725173444e-05,1341 2.403900543868076e-05,1342 8.416523996856995e-06,1343 -2.933492032752838e-051344 ],1345 [1346 0.0,1347 0.0,1348 0.0,1349 -7.816145080141723e-05,1350 1.2711129784293007e-05,1351 -1.1673649169097189e-05,1352 -0.000235616054851561781353 ],1354 [1355 0.0,1356 0.0,1357 0.0,1358 0.0,1359 0.0,1360 0.0,1361 -0.0048099867999553681362 ]1363 ]1364 },1365 "mean_flip_rate": [1366 [1367 0.0,1368 0.0,1369 0.0,1370 0.0,1371 0.0,1372 0.0,1373 1.01374 ],1375 [1376 0.0,1377 0.0,1378 0.0,1379 0.0,1380 0.0,1381 0.0,1382 0.01383 ],1384 [1385 0.0,1386 0.0,1387 0.0,1388 0.0,1389 0.0,1390 0.0,1391 0.01392 ],1393 [1394 0.0,1395 0.0,1396 0.0,1397 0.0,1398 0.0,1399 0.0,1400 0.01401 ]1402 ]1403 },1404 "mlp_out": {1405 "s0": {1406 "flip_rate": [1407 [1408 0.0,1409 0.0,1410 0.0,1411 0.0,1412 0.0,1413 0.0,1414 1.01415 ],1416 [1417 0.0,1418 0.0,1419 0.0,1420 0.0,1421 0.0,1422 0.0,1423 1.01424 ],1425 [1426 0.0,1427 0.0,1428 0.0,1429 0.0,1430 0.0,1431 0.0,1432 0.81428569555282591433 ],1434 [1435 0.0,1436 0.0,1437 0.0,1438 0.0,1439 0.0,1440 0.0,1441 0.406122446060180661442 ]1443 ],1444 "logit_recovery": [1445 [1446 0.0,1447 0.0,1448 0.0,1449 -0.00376747059635818,1450 0.0001691565994406119,1451 3.0771909678151133e-06,1452 0.98868495225906371453 ],1454 [1455 0.0,1456 0.0,1457 0.0,1458 -0.004375732969492674,1459 -6.94971313350834e-05,1460 -2.2410305973608047e-05,1461 0.7163521051406861462 ],1463 [1464 0.0,1465 0.0,1466 0.0,1467 -5.74394543946255e-05,1468 2.9751297915936448e-05,1469 -2.8927430321346037e-05,1470 0.59569227695465091471 ],1472 [1473 0.0,1474 0.0,1475 0.0,1476 0.0,1477 0.0,1478 0.0,1479 0.50334531068801881480 ]1481 ]1482 },1483 "s1": {1484 "flip_rate": [1485 [1486 0.0,1487 0.0,1488 0.0,1489 0.0,1490 0.0,1491 0.0,1492 1.01493 ],1494 [1495 0.0,1496 0.0,1497 0.0,1498 0.0,1499 0.0,1500 0.0,1501 0.97551017999649051502 ],1503 [1504 0.0,1505 0.0,1506 0.0,1507 0.0,1508 0.0,1509 0.0,1510 0.94489794969558721511 ],1512 [1513 0.0,1514 0.0,1515 0.0,1516 0.0,1517 0.0,1518 0.0,1519 0.01520 ]1521 ],1522 "logit_recovery": [1523 [1524 0.0,1525 0.0,1526 0.0,1527 0.0005149293574504554,1528 0.000171999228768982,1529 1.244249716592094e-07,1530 0.99418604373931881531 ],1532 [1533 0.0,1534 0.0,1535 0.0,1536 -0.00022421055473387241,1537 6.864670285722241e-05,1538 1.3663714071299182e-06,1539 0.85900801420211791540 ],1541 [1542 0.0,1543 0.0,1544 0.0,1545 -0.00016121681255754083,1546 -2.486043558747042e-05,1547 3.6747774174727965e-06,1548 0.6204722523689271549 ],1550 [1551 0.0,1552 0.0,1553 0.0,1554 0.0,1555 0.0,1556 0.0,1557 0.249156743288040161558 ]1559 ]1560 },1561 "s2": {1562 "flip_rate": [1563 [1564 0.0,1565 0.0,1566 0.0,1567 0.0,1568 0.0,1569 0.0,1570 1.01571 ],1572 [1573 0.0,1574 0.0,1575 0.0,1576 0.0,1577 0.0,1578 0.0,1579 0.73265308141708371580 ],1581 [1582 0.0,1583 0.0,1584 0.0,1585 0.0,1586 0.0,1587 0.0,1588 0.7224490046501161589 ],1590 [1591 0.0,1592 0.0,1593 0.0,1594 0.0,1595 0.0,1596 0.0,1597 0.114285714924335481598 ]1599 ],1600 "logit_recovery": [1601 [1602 0.0,1603 0.0,1604 0.0,1605 0.0005150199285708368,1606 -3.605287474783836e-06,1607 1.780065531420405e-06,1608 0.99598747491836551609 ],1610 [1611 0.0,1612 0.0,1613 0.0,1614 -0.00022224811255000532,1615 -4.047781476401724e-05,1616 2.9307227578101447e-06,1617 0.6566138267517091618 ],1619 [1620 0.0,1621 0.0,1622 0.0,1623 -4.242335489834659e-05,1624 -7.299037406482967e-06,1625 5.311194854584755e-06,1626 0.61601710319519041627 ],1628 [1629 0.0,1630 0.0,1631 0.0,1632 0.0,1633 0.0,1634 0.0,1635 0.37208849191665651636 ]1637 ]1638 },1639 "mean_flip_rate": [1640 [1641 0.0,1642 0.0,1643 0.0,1644 0.0,1645 0.0,1646 0.0,1647 1.01648 ],1649 [1650 0.0,1651 0.0,1652 0.0,1653 0.0,1654 0.0,1655 0.0,1656 0.9027210871378581657 ],1658 [1659 0.0,1660 0.0,1661 0.0,1662 0.0,1663 0.0,1664 0.0,1665 0.82721088329950961666 ],1667 [1668 0.0,1669 0.0,1670 0.0,1671 0.0,1672 0.0,1673 0.0,1674 0.173469386994838711675 ]1676 ]1677 }1678 },1679 "ablation": {1680 "resid_post": {1681 "s0": [1682 [1683 1.0,1684 1.0,1685 1.0,1686 1.0,1687 1.0,1688 1.0,1689 0.01690 ],1691 [1692 1.0,1693 1.0,1694 1.0,1695 1.0,1696 1.0,1697 1.0,1698 0.01699 ],1700 [1701 1.0,1702 1.0,1703 1.0,1704 1.0,1705 1.0,1706 1.0,1707 0.087755098938941961708 ],1709 [1710 1.0,1711 1.0,1712 1.0,1713 1.0,1714 1.0,1715 1.0,1716 0.085714288055896761717 ]1718 ],1719 "s1": [1720 [1721 1.0,1722 1.0,1723 1.0,1724 1.0,1725 1.0,1726 1.0,1727 0.089795917272567751728 ],1729 [1730 1.0,1731 1.0,1732 1.0,1733 1.0,1734 1.0,1735 1.0,1736 0.03673469275236131737 ],1738 [1739 1.0,1740 1.0,1741 1.0,1742 1.0,1743 1.0,1744 1.0,1745 0.079591833055019381746 ],1747 [1748 1.0,1749 1.0,1750 1.0,1751 1.0,1752 1.0,1753 1.0,1754 0.085714288055896761755 ]1756 ],1757 "s2": [1758 [1759 1.0,1760 1.0,1761 1.0,1762 1.0,1763 1.0,1764 1.0,1765 0.018367346376180651766 ],1767 [1768 1.0,1769 1.0,1770 1.0,1771 1.0,1772 1.0,1773 1.0,1774 0.087755098938941961775 ],1776 [1777 1.0,1778 1.0,1779 1.0,1780 1.0,1781 1.0,1782 1.0,1783 0.091836735606193541784 ],1785 [1786 1.0,1787 1.0,1788 1.0,1789 1.0,1790 1.0,1791 1.0,1792 0.085714288055896761793 ]1794 ]1795 },1796 "attn_out": {1797 "s0": [1798 [1799 1.0,1800 1.0,1801 1.0,1802 1.0,1803 1.0,1804 1.0,1805 0.116326533257961271806 ],1807 [1808 1.0,1809 1.0,1810 1.0,1811 1.0,1812 1.0,1813 1.0,1814 1.01815 ],1816 [1817 1.0,1818 1.0,1819 1.0,1820 1.0,1821 1.0,1822 1.0,1823 1.01824 ],1825 [1826 1.0,1827 1.0,1828 1.0,1829 1.0,1830 1.0,1831 1.0,1832 1.01833 ]1834 ],1835 "s1": [1836 [1837 1.0,1838 1.0,1839 1.0,1840 1.0,1841 1.0,1842 1.0,1843 0.171428576111793521844 ],1845 [1846 1.0,1847 1.0,1848 1.0,1849 1.0,1850 1.0,1851 1.0,1852 1.01853 ],1854 [1855 1.0,1856 1.0,1857 1.0,1858 1.0,1859 1.0,1860 1.0,1861 1.01862 ],1863 [1864 1.0,1865 1.0,1866 1.0,1867 1.0,1868 1.0,1869 1.0,1870 1.01871 ]1872 ],1873 "s2": [1874 [1875 1.0,1876 1.0,1877 1.0,1878 1.0,1879 1.0,1880 1.0,1881 0.089795917272567751882 ],1883 [1884 1.0,1885 1.0,1886 1.0,1887 1.0,1888 1.0,1889 1.0,1890 1.01891 ],1892 [1893 1.0,1894 1.0,1895 1.0,1896 1.0,1897 1.0,1898 1.0,1899 1.01900 ],1901 [1902 1.0,1903 1.0,1904 1.0,1905 1.0,1906 1.0,1907 1.0,1908 1.01909 ]1910 ]1911 },1912 "mlp_out": {1913 "s0": [1914 [1915 1.0,1916 1.0,1917 1.0,1918 1.0,1919 1.0,1920 1.0,1921 0.126530617475509641922 ],1923 [1924 1.0,1925 1.0,1926 1.0,1927 1.0,1928 1.0,1929 1.0,1930 0.91428571939468381931 ],1932 [1933 1.0,1934 1.0,1935 1.0,1936 1.0,1937 1.0,1938 1.0,1939 1.01940 ],1941 [1942 1.0,1943 1.0,1944 1.0,1945 1.0,1946 1.0,1947 1.0,1948 1.01949 ]1950 ],1951 "s1": [1952 [1953 1.0,1954 1.0,1955 1.0,1956 1.0,1957 1.0,1958 1.0,1959 0.39183673262596131960 ],1961 [1962 1.0,1963 1.0,1964 1.0,1965 1.0,1966 1.0,1967 1.0,1968 1.01969 ],1970 [1971 1.0,1972 1.0,1973 1.0,1974 1.0,1975 1.0,1976 1.0,1977 1.01978 ],1979 [1980 1.0,1981 1.0,1982 1.0,1983 1.0,1984 1.0,1985 1.0,1986 1.01987 ]1988 ],1989 "s2": [1990 [1991 1.0,1992 1.0,1993 1.0,1994 1.0,1995 1.0,1996 1.0,1997 0.110204078257083891998 ],1999 [2000 1.0,2001 1.0,2002 1.0,2003 1.0,2004 1.0,2005 1.0,2006 1.02007 ],2008 [2009 1.0,2010 1.0,2011 1.0,2012 1.0,2013 1.0,2014 1.0,2015 1.02016 ],2017 [2018 1.0,2019 1.0,2020 1.0,2021 1.0,2022 1.0,2023 1.0,2024 1.02025 ]2026 ]2027 }2028 },2029 "heads": {2030 "omega_token": {2031 "s0": [2032 [2033 0.0,2034 0.0,2035 0.0,2036 0.02037 ],2038 [2039 0.0,2040 0.0,2041 0.0,2042 0.02043 ],2044 [2045 0.0,2046 0.0,2047 0.0,2048 0.02049 ],2050 [2051 0.0,2052 0.0,2053 0.0,2054 0.02055 ]2056 ],2057 "s1": [2058 [2059 0.0,2060 0.0,2061 0.0,2062 0.02063 ],2064 [2065 0.0,2066 0.0,2067 0.0,2068 0.02069 ],2070 [2071 0.0,2072 0.0,2073 0.0,2074 0.02075 ],2076 [2077 0.0,2078 0.0,2079 0.0,2080 0.02081 ]2082 ],2083 "s2": [2084 [2085 0.0,2086 0.0,2087 0.0,2088 0.02089 ],2090 [2091 0.0,2092 0.0,2093 0.0,2094 0.02095 ],2096 [2097 0.0,2098 0.0,2099 0.0,2100 0.02101 ],2102 [2103 0.0,2104 0.0,2105 0.0,2106 0.02107 ]2108 ]2109 },2110 "read_position": {2111 "s0": [2112 [2113 0.9346938729286194,2114 0.4734693765640259,2115 0.0,2116 0.02117 ],2118 [2119 0.0,2120 0.0,2121 0.0,2122 0.02123 ],2124 [2125 0.0,2126 0.0,2127 0.0,2128 0.02129 ],2130 [2131 0.0,2132 0.0,2133 0.0,2134 0.02135 ]2136 ],2137 "s1": [2138 [2139 0.9163265228271484,2140 0.0,2141 0.0,2142 0.02143 ],2144 [2145 0.0,2146 0.0,2147 0.0,2148 0.02149 ],2150 [2151 0.0,2152 0.0,2153 0.0,2154 0.02155 ],2156 [2157 0.0,2158 0.0,2159 0.0,2160 0.02161 ]2162 ],2163 "s2": [2164 [2165 0.0,2166 0.0,2167 0.11428571492433548,2168 0.02169 ],2170 [2171 0.0,2172 0.0,2173 0.0,2174 0.02175 ],2176 [2177 0.0,2178 0.0,2179 0.0,2180 0.02181 ],2182 [2183 0.0,2184 0.0,2185 0.0,2186 0.02187 ]2188 ]2189 }2190 },2191 "reproducibility": {2192 "resid_post_s0_repeat_max_abs_diff": 0.0,2193 "attn_out_s0_repeat_max_abs_diff": 0.0,2194 "mlp_out_s0_repeat_max_abs_diff": 0.0,2195 "resid_post_s1_repeat_max_abs_diff": 0.0,2196 "attn_out_s1_repeat_max_abs_diff": 0.0,2197 "mlp_out_s1_repeat_max_abs_diff": 0.0,2198 "resid_post_s2_repeat_max_abs_diff": 0.0,2199 "attn_out_s2_repeat_max_abs_diff": 0.0,2200 "mlp_out_s2_repeat_max_abs_diff": 0.0,2201 "resid_post_across_seed_pearson_min": 0.9999999999999999,2202 "attn_out_across_seed_pearson_min": 0.9999999999999999,2203 "mlp_out_across_seed_pearson_min": 0.96423624483102982204 },2205 "baselines": {2206 "s0": {2207 "clean_acc_on_clean_answer": 1.0,2208 "corrupt_run_acc_on_clean_answer(sham)": 0.02209 },2210 "s1": {2211 "clean_acc_on_clean_answer": 1.0,2212 "corrupt_run_acc_on_clean_answer(sham)": 0.02213 },2214 "s2": {2215 "clean_acc_on_clean_answer": 1.0,2216 "corrupt_run_acc_on_clean_answer(sham)": 0.02217 }2218 }2219 },2220 "across_seeds": {2221 "target_iid": {2222 "donor": {2223 "n_seeds": 3,2224 "mean": 1.0,2225 "sd": 0.0,2226 "min": 1.0,2227 "max": 1.02228 },2229 "recipient": {2230 "n_seeds": 3,2231 "mean": 0.09200000000000001,2232 "sd": 0.05288667128870941,2233 "min": 0.059,2234 "max": 0.1532235 }2236 },2237 "control_iid": {2238 "donor": {2239 "n_seeds": 3,2240 "mean": 1.0,2241 "sd": 0.0,2242 "min": 1.0,2243 "max": 1.02244 },2245 "recipient": {2246 "n_seeds": 3,2247 "mean": 1.0,2248 "sd": 0.0,2249 "min": 1.0,2250 "max": 1.02251 }2252 },2253 "comp_heldout": {2254 "donor": {2255 "n_seeds": 3,2256 "mean": 1.0,2257 "sd": 0.0,2258 "min": 1.0,2259 "max": 1.02260 },2261 "recipient": {2262 "n_seeds": 3,2263 "mean": 0.4503333333333333,2264 "sd": 0.031895663237081824,2265 "min": 0.429,2266 "max": 0.4872267 }2268 },2269 "len_ood": {2270 "donor": {2271 "n_seeds": 3,2272 "mean": 0.08600000000000001,2273 "sd": 0.017320508075688773,2274 "min": 0.076,2275 "max": 0.1062276 },2277 "recipient": {2278 "n_seeds": 3,2279 "mean": 0.09066666666666666,2280 "sd": 0.015534906930308057,2281 "min": 0.078,2282 "max": 0.1082283 }2284 }2285 },2286 "across_seeds_omega_step": {2287 "donor": {2288 "n_seeds": 3,2289 "mean": 1.0,2290 "sd": 0.0,2291 "min": 1.0,2292 "max": 1.02293 },2294 "recipient": {2295 "n_seeds": 3,2296 "mean": 0.06166666666666667,2297 "sd": 0.0697160909212022,2298 "min": 0.017,2299 "max": 0.1422300 }2301 },2302 "elapsed_seconds": 275.54194569587712303}