Repository Public Research preview
1{2 "tasks": {3 "antonym": {4 "n_queries": 12,5 "hash": "22992056ec21621c"6 },7 "capital": {8 "n_queries": 12,9 "hash": "0ce7fe08025fd9c5"10 },11 "morph": {12 "n_queries": 12,13 "hash": "313569abbc843a4f"14 },15 "addthree": {16 "n_queries": 12,17 "hash": "63af249180d379b8"18 }19 },20 "models": {21 "qwen1.5b": {22 "repo": "mlx-community/Qwen2.5-1.5B-Instruct-bf16",23 "family": "Qwen2.5",24 "license": "Apache-2.0",25 "n_layers": 28,26 "hidden": 1536,27 "precision": "bf16",28 "localization": {29 "layers": [30 2,31 7,32 11,33 16,34 20,35 2536 ],37 "tasks": {38 "antonym": {39 "zero_shot": 0.0,40 "few_shot": 1.0,41 "sufficiency_by_layer": {42 "2": 0.0,43 "7": 0.08333333333333333,44 "11": 0.16666666666666666,45 "16": 0.75,46 "20": 0.5833333333333334,47 "25": 0.333333333333333348 },49 "best_layer": 16,50 "necessity_fewshot_minus_direction": 0.9166666666666666,51 "random_vector_same_norm_at_best": 0.3333333333333333,52 "vector_norm": 38.54524993896484453 },54 "capital": {55 "zero_shot": 0.0,56 "few_shot": 1.0,57 "sufficiency_by_layer": {58 "2": 0.0,59 "7": 0.0,60 "11": 0.08333333333333333,61 "16": 0.5833333333333334,62 "20": 1.0,63 "25": 0.7564 },65 "best_layer": 20,66 "necessity_fewshot_minus_direction": 1.0,67 "random_vector_same_norm_at_best": 0.0,68 "vector_norm": 54.3085136413574269 },70 "morph": {71 "zero_shot": 0.0,72 "few_shot": 0.25,73 "sufficiency_by_layer": {74 "2": 0.0,75 "7": 0.0,76 "11": 0.0,77 "16": 0.0,78 "20": 0.0,79 "25": 0.080 },81 "best_layer": 2,82 "necessity_fewshot_minus_direction": 0.25,83 "random_vector_same_norm_at_best": 0.0,84 "vector_norm": 12.20187091827392685 },86 "addthree": {87 "zero_shot": 0.0,88 "few_shot": 0.9166666666666666,89 "sufficiency_by_layer": {90 "2": 0.0,91 "7": 0.0,92 "11": 0.0,93 "16": 0.0,94 "20": 0.0,95 "25": 0.096 },97 "best_layer": 2,98 "necessity_fewshot_minus_direction": 0.9166666666666666,99 "random_vector_same_norm_at_best": 0.0,100 "vector_norm": 12.588272094726562101 }102 }103 }104 },105 "qwen3b": {106 "repo": "mlx-community/Qwen2.5-3B-Instruct-bf16",107 "family": "Qwen2.5",108 "license": "Qwen Research License",109 "n_layers": 36,110 "hidden": 2048,111 "precision": "bf16",112 "localization": {113 "layers": [114 2,115 8,116 14,117 21,118 27,119 33120 ],121 "tasks": {122 "antonym": {123 "zero_shot": 0.25,124 "few_shot": 0.9166666666666666,125 "sufficiency_by_layer": {126 "2": 0.3333333333333333,127 "8": 0.25,128 "14": 0.3333333333333333,129 "21": 0.6666666666666666,130 "27": 0.5833333333333334,131 "33": 0.16666666666666666132 },133 "best_layer": 21,134 "necessity_fewshot_minus_direction": 1.0,135 "random_vector_same_norm_at_best": 0.16666666666666666,136 "vector_norm": 45.75545883178711137 },138 "capital": {139 "zero_shot": 0.0,140 "few_shot": 1.0,141 "sufficiency_by_layer": {142 "2": 0.0,143 "8": 0.0,144 "14": 0.08333333333333333,145 "21": 0.0,146 "27": 0.8333333333333334,147 "33": 0.6666666666666666148 },149 "best_layer": 27,150 "necessity_fewshot_minus_direction": 1.0,151 "random_vector_same_norm_at_best": 0.0,152 "vector_norm": 75.41123962402344153 },154 "morph": {155 "zero_shot": 0.0,156 "few_shot": 0.3333333333333333,157 "sufficiency_by_layer": {158 "2": 0.0,159 "8": 0.0,160 "14": 0.0,161 "21": 0.0,162 "27": 0.0,163 "33": 0.0164 },165 "best_layer": 2,166 "necessity_fewshot_minus_direction": 0.3333333333333333,167 "random_vector_same_norm_at_best": 0.0,168 "vector_norm": 6.109277725219727169 },170 "addthree": {171 "zero_shot": 0.0,172 "few_shot": 1.0,173 "sufficiency_by_layer": {174 "2": 0.0,175 "8": 0.0,176 "14": 0.0,177 "21": 0.0,178 "27": 0.0,179 "33": 0.0180 },181 "best_layer": 2,182 "necessity_fewshot_minus_direction": 1.0,183 "random_vector_same_norm_at_best": 0.0,184 "vector_norm": 5.987315654754639185 }186 }187 }188 }189 },190 "pairs": {191 "qwen3b->qwen1.5b": {192 "donor": "qwen3b",193 "recipient": "qwen1.5b",194 "tasks": {195 "antonym": {196 "recipient_zero": 0.0,197 "recipient_few": 1.0,198 "recipient_own_vector": 0.75,199 "donor_layer": 21,200 "conditions": {201 "procrustes|L16|a1.0": {202 "accuracy": 0.08333333333333333,203 "gain_vs_zero": {204 "estimate": 0.08333333333333333,205 "low": 0.0,206 "high": 0.25,207 "level": 0.95,208 "n_items": 12,209 "n_clusters": 12,210 "n_boot": 300211 },212 "mapped_norm": 30.465124130249023,213 "outputs_sample": [214 " 1\n",215 " 1\n",216 " 1\n",217 " Please provide a question for me",218 " 1\n"219 ]220 },221 "ridge|L16|a1.0": {222 "accuracy": 0.25,223 "gain_vs_zero": {224 "estimate": 0.25,225 "low": 0.08333333333333333,226 "high": 0.5,227 "level": 0.95,228 "n_items": 12,229 "n_clusters": 12,230 "n_boot": 300231 },232 "mapped_norm": 16.41504669189453,233 "outputs_sample": [234 " 1\n",235 " 1\n",236 " 1\n",237 " answer\n",238 " 1\n"239 ]240 },241 "cca|L16|a1.0": {242 "accuracy": 0.25,243 "gain_vs_zero": {244 "estimate": 0.25,245 "low": 0.08333333333333333,246 "high": 0.5,247 "level": 0.95,248 "n_items": 12,249 "n_clusters": 12,250 "n_boot": 300251 },252 "mapped_norm": 16.53496551513672,253 "outputs_sample": [254 " 1\n",255 " 1\n",256 " 1\n",257 " answer\n",258 " 1\n"259 ]260 },261 "random_orth|L16|a1.0": {262 "accuracy": 0.0,263 "gain_vs_zero": {264 "estimate": 0.0,265 "low": 0.0,266 "high": 0.0,267 "level": 0.95,268 "n_items": 12,269 "n_clusters": 12,270 "n_boot": 300271 },272 "mapped_norm": 30.173856735229492,273 "outputs_sample": [274 " wide\n",275 " add\n",276 " safe\n",277 " \"I'm sorry, I",278 " raw\n\n"279 ]280 },281 "ctl_random_norm_matched|L16": {282 "accuracy": 0.0,283 "gain_vs_zero": {284 "estimate": 0.0,285 "low": 0.0,286 "high": 0.0,287 "level": 0.95,288 "n_items": 12,289 "n_clusters": 12,290 "n_boot": 300291 }292 },293 "ctl_unrelated_task_vector(capital)|L16": {294 "accuracy": 0.25,295 "gain_vs_zero": {296 "estimate": 0.25,297 "low": 0.08333333333333333,298 "high": 0.5,299 "level": 0.95,300 "n_items": 12,301 "n_clusters": 12,302 "n_boot": 300303 }304 }305 }306 },307 "capital": {308 "recipient_zero": 0.0,309 "recipient_few": 1.0,310 "recipient_own_vector": 1.0,311 "donor_layer": 27,312 "conditions": {313 "procrustes|L20|a1.0": {314 "accuracy": 0.0,315 "gain_vs_zero": {316 "estimate": 0.0,317 "low": 0.0,318 "high": 0.0,319 "level": 0.95,320 "n_items": 12,321 "n_clusters": 12,322 "n_boot": 300323 },324 "mapped_norm": 51.595008850097656,325 "outputs_sample": [326 " 10000",327 " Ukraine is a country located in",328 " 1\n\n",329 " 1\n\n",330 " 1\n\n"331 ]332 },333 "ridge|L20|a1.0": {334 "accuracy": 0.08333333333333333,335 "gain_vs_zero": {336 "estimate": 0.08333333333333333,337 "low": 0.0,338 "high": 0.25,339 "level": 0.95,340 "n_items": 12,341 "n_clusters": 12,342 "n_boot": 300343 },344 "mapped_norm": 30.142295837402344,345 "outputs_sample": [346 " The capital of Cuba is Havana",347 " Russia\n\n",348 " The capital of Norway is Oslo",349 " 1\n\n",350 " Singapore\n\n"351 ]352 },353 "cca|L20|a1.0": {354 "accuracy": 0.08333333333333333,355 "gain_vs_zero": {356 "estimate": 0.08333333333333333,357 "low": 0.0,358 "high": 0.25,359 "level": 0.95,360 "n_items": 12,361 "n_clusters": 12,362 "n_boot": 300363 },364 "mapped_norm": 30.298126220703125,365 "outputs_sample": [366 " The capital of Cuba is Havana",367 " Russia\n\n",368 " The capital of Norway is Oslo",369 " 1\n\n",370 " Singapore\n\n"371 ]372 },373 "random_orth|L20|a1.0": {374 "accuracy": 0.0,375 "gain_vs_zero": {376 "estimate": 0.0,377 "low": 0.0,378 "high": 0.0,379 "level": 0.95,380 "n_items": 12,381 "n_clusters": 12,382 "n_boot": 300383 },384 "mapped_norm": 53.39498519897461,385 "outputs_sample": [386 " The capital of Cuba is Havana",387 " Russia\n",388 " The capital of Norway is Oslo",389 " The capital of Colombia is Bog",390 " Singapore\n"391 ]392 },393 "ctl_random_norm_matched|L20": {394 "accuracy": 0.0,395 "gain_vs_zero": {396 "estimate": 0.0,397 "low": 0.0,398 "high": 0.0,399 "level": 0.95,400 "n_items": 12,401 "n_clusters": 12,402 "n_boot": 300403 }404 },405 "ctl_unrelated_task_vector(antonym)|L20": {406 "accuracy": 0.08333333333333333,407 "gain_vs_zero": {408 "estimate": 0.08333333333333333,409 "low": 0.0,410 "high": 0.25,411 "level": 0.95,412 "n_items": 12,413 "n_clusters": 12,414 "n_boot": 300415 }416 },417 "procrustes|L21|a1.0": {418 "accuracy": 0.0,419 "gain_vs_zero": {420 "estimate": 0.0,421 "low": 0.0,422 "high": 0.0,423 "level": 0.95,424 "n_items": 12,425 "n_clusters": 12,426 "n_boot": 300427 },428 "mapped_norm": 58.942562103271484,429 "outputs_sample": [430 " Cuba is a country located in",431 " Russia\n",432 " Norway is a country located in",433 " Colombia is a country located in",434 " Malaysia is a country located in"435 ]436 },437 "ridge|L21|a1.0": {438 "accuracy": 0.0,439 "gain_vs_zero": {440 "estimate": 0.0,441 "low": 0.0,442 "high": 0.0,443 "level": 0.95,444 "n_items": 12,445 "n_clusters": 12,446 "n_boot": 300447 },448 "mapped_norm": 35.16560363769531,449 "outputs_sample": [450 " The capital of Cuba is Havana",451 " Russia\n\n",452 " The capital of Norway is Oslo",453 " The capital of Colombia is Bog",454 " Singapore\n\n"455 ]456 },457 "cca|L21|a1.0": {458 "accuracy": 0.0,459 "gain_vs_zero": {460 "estimate": 0.0,461 "low": 0.0,462 "high": 0.0,463 "level": 0.95,464 "n_items": 12,465 "n_clusters": 12,466 "n_boot": 300467 },468 "mapped_norm": 35.3463134765625,469 "outputs_sample": [470 " The capital of Cuba is Havana",471 " Russia\n\n",472 " The capital of Norway is Oslo",473 " The capital of Colombia is Bog",474 " Singapore\n\n"475 ]476 },477 "random_orth|L21|a1.0": {478 "accuracy": 0.0,479 "gain_vs_zero": {480 "estimate": 0.0,481 "low": 0.0,482 "high": 0.0,483 "level": 0.95,484 "n_items": 12,485 "n_clusters": 12,486 "n_boot": 300487 },488 "mapped_norm": 61.71129608154297,489 "outputs_sample": [490 " I'm sorry, but I",491 " Russia\n",492 " The capital of Norway is Oslo",493 " The capital of Colombia is Bog",494 " I'm sorry, I'm"495 ]496 },497 "ctl_random_norm_matched|L21": {498 "accuracy": 0.0,499 "gain_vs_zero": {500 "estimate": 0.0,501 "low": 0.0,502 "high": 0.0,503 "level": 0.95,504 "n_items": 12,505 "n_clusters": 12,506 "n_boot": 300507 }508 },509 "ctl_unrelated_task_vector(antonym)|L21": {510 "accuracy": 0.0,511 "gain_vs_zero": {512 "estimate": 0.0,513 "low": 0.0,514 "high": 0.0,515 "level": 0.95,516 "n_items": 12,517 "n_clusters": 12,518 "n_boot": 300519 }520 }521 }522 },523 "morph": {524 "recipient_zero": 0.0,525 "recipient_few": 0.25,526 "recipient_own_vector": 0.0,527 "donor_layer": 2,528 "conditions": {529 "procrustes|L2|a1.0": {530 "accuracy": 0.0,531 "gain_vs_zero": {532 "estimate": 0.0,533 "low": 0.0,534 "high": 0.0,535 "level": 0.95,536 "n_items": 12,537 "n_clusters": 12,538 "n_boot": 300539 },540 "mapped_norm": 7.246042251586914,541 "outputs_sample": [542 " The word \"repa\"",543 " Sesalun is a word",544 " The word \"panaf\"",545 " \"The quick brown fox jumps",546 " 1\n"547 ]548 },549 "ridge|L2|a1.0": {550 "accuracy": 0.0,551 "gain_vs_zero": {552 "estimate": 0.0,553 "low": 0.0,554 "high": 0.0,555 "level": 0.95,556 "n_items": 12,557 "n_clusters": 12,558 "n_boot": 300559 },560 "mapped_norm": 3.5472207069396973,561 "outputs_sample": [562 " 1\n",563 " Sesalun is a word",564 " The word \"panaf\"",565 " \"The quick brown fox jumps",566 " 1\n"567 ]568 },569 "cca|L2|a1.0": {570 "accuracy": 0.0,571 "gain_vs_zero": {572 "estimate": 0.0,573 "low": 0.0,574 "high": 0.0,575 "level": 0.95,576 "n_items": 12,577 "n_clusters": 12,578 "n_boot": 300579 },580 "mapped_norm": 6.425337791442871,581 "outputs_sample": [582 " 1\n",583 " Sesalun is a word",584 " The word \"panaf\"",585 " \"The quick brown fox jumps",586 " 1\n"587 ]588 },589 "random_orth|L2|a1.0": {590 "accuracy": 0.0,591 "gain_vs_zero": {592 "estimate": 0.0,593 "low": 0.0,594 "high": 0.0,595 "level": 0.95,596 "n_items": 12,597 "n_clusters": 12,598 "n_boot": 300599 },600 "mapped_norm": 7.464280128479004,601 "outputs_sample": [602 " 1\n",603 " Sesalun is a word",604 " The word \"panaf\"",605 " \"The quick brown fox jumps",606 " 1\n"607 ]608 },609 "ctl_random_norm_matched|L2": {610 "accuracy": 0.0,611 "gain_vs_zero": {612 "estimate": 0.0,613 "low": 0.0,614 "high": 0.0,615 "level": 0.95,616 "n_items": 12,617 "n_clusters": 12,618 "n_boot": 300619 }620 },621 "ctl_unrelated_task_vector(antonym)|L2": {622 "accuracy": 0.0,623 "gain_vs_zero": {624 "estimate": 0.0,625 "low": 0.0,626 "high": 0.0,627 "level": 0.95,628 "n_items": 12,629 "n_clusters": 12,630 "n_boot": 300631 }632 }633 }634 },635 "addthree": {636 "recipient_zero": 0.0,637 "recipient_few": 0.9166666666666666,638 "recipient_own_vector": 0.0,639 "donor_layer": 2,640 "conditions": {641 "procrustes|L2|a1.0": {642 "accuracy": 0.0,643 "gain_vs_zero": {644 "estimate": 0.0,645 "low": 0.0,646 "high": 0.0,647 "level": 0.95,648 "n_items": 12,649 "n_clusters": 12,650 "n_boot": 300651 },652 "mapped_norm": 7.081792831420898,653 "outputs_sample": [654 " 12\n",655 " 3\n\n",656 " \"FizzBuzz\"\n",657 " \"58\"\n",658 " \"Fizz\"\n\n"659 ]660 },661 "ridge|L2|a1.0": {662 "accuracy": 0.0,663 "gain_vs_zero": {664 "estimate": 0.0,665 "low": 0.0,666 "high": 0.0,667 "level": 0.95,668 "n_items": 12,669 "n_clusters": 12,670 "n_boot": 300671 },672 "mapped_norm": 3.283189296722412,673 "outputs_sample": [674 " 12\n",675 " \"63\"\n",676 " \"FizzBuzz\"\n",677 " \"Fizz\"\n\n",678 " \"Fizz\"\n\n"679 ]680 },681 "cca|L2|a1.0": {682 "accuracy": 0.0,683 "gain_vs_zero": {684 "estimate": 0.0,685 "low": 0.0,686 "high": 0.0,687 "level": 0.95,688 "n_items": 12,689 "n_clusters": 12,690 "n_boot": 300691 },692 "mapped_norm": 6.4316816329956055,693 "outputs_sample": [694 " 12\n",695 " 3\n",696 " \"FizzBuzz\"\n",697 " \"Fizz\"\n\n",698 " \"Fizz\"\n\n"699 ]700 },701 "random_orth|L2|a1.0": {702 "accuracy": 0.0,703 "gain_vs_zero": {704 "estimate": 0.0,705 "low": 0.0,706 "high": 0.0,707 "level": 0.95,708 "n_items": 12,709 "n_clusters": 12,710 "n_boot": 300711 },712 "mapped_norm": 7.317880630493164,713 "outputs_sample": [714 " 12\n",715 " \"63\"\n",716 " \"FizzBuzz\"\n",717 " \"Fizz\"\n\n",718 " \"Fizz\"\n\n"719 ]720 },721 "ctl_random_norm_matched|L2": {722 "accuracy": 0.0,723 "gain_vs_zero": {724 "estimate": 0.0,725 "low": 0.0,726 "high": 0.0,727 "level": 0.95,728 "n_items": 12,729 "n_clusters": 12,730 "n_boot": 300731 }732 },733 "ctl_unrelated_task_vector(antonym)|L2": {734 "accuracy": 0.0,735 "gain_vs_zero": {736 "estimate": 0.0,737 "low": 0.0,738 "high": 0.0,739 "level": 0.95,740 "n_items": 12,741 "n_clusters": 12,742 "n_boot": 300743 }744 }745 }746 }747 },748 "maps": {749 "antonym|L21->L16|procrustes": {750 "r2_heldout_text": 0.8879651847238482,751 "n_params": 3147264752 },753 "antonym|L21->L16|ridge": {754 "r2_heldout_text": 0.9999780218151069,755 "n_params": 3147264756 },757 "antonym|L21->L16|cca": {758 "r2_heldout_text": 1.0,759 "n_params": 3147264760 },761 "antonym|L21->L16|random_orth": {762 "r2_heldout_text": -0.7739848588177365,763 "n_params": 3147264764 },765 "capital|L27->L20|procrustes": {766 "r2_heldout_text": 0.8628232567523446,767 "n_params": 3147264768 },769 "capital|L27->L20|ridge": {770 "r2_heldout_text": 0.9999881340059268,771 "n_params": 3147264772 },773 "capital|L27->L20|cca": {774 "r2_heldout_text": 1.0,775 "n_params": 3147264776 },777 "capital|L27->L20|random_orth": {778 "r2_heldout_text": -0.7724365231388266,779 "n_params": 3147264780 },781 "capital|L27->L21|procrustes": {782 "r2_heldout_text": 0.8532479095046922,783 "n_params": 3147264784 },785 "capital|L27->L21|ridge": {786 "r2_heldout_text": 0.9999879738580739,787 "n_params": 3147264788 },789 "capital|L27->L21|cca": {790 "r2_heldout_text": 1.0,791 "n_params": 3147264792 },793 "capital|L27->L21|random_orth": {794 "r2_heldout_text": -0.7659151922938061,795 "n_params": 3147264796 },797 "morph|L2->L2|procrustes": {798 "r2_heldout_text": 0.9133215800216044,799 "n_params": 3147264800 },801 "morph|L2->L2|ridge": {802 "r2_heldout_text": 0.9829605177270799,803 "n_params": 3147264804 },805 "morph|L2->L2|cca": {806 "r2_heldout_text": 1.0,807 "n_params": 3147264808 },809 "morph|L2->L2|random_orth": {810 "r2_heldout_text": -0.8259323728704582,811 "n_params": 3147264812 },813 "addthree|L2->L2|procrustes": {814 "r2_heldout_text": 0.9133215800216044,815 "n_params": 3147264816 },817 "addthree|L2->L2|ridge": {818 "r2_heldout_text": 0.9829605177270799,819 "n_params": 3147264820 },821 "addthree|L2->L2|cca": {822 "r2_heldout_text": 1.0,823 "n_params": 3147264824 },825 "addthree|L2->L2|random_orth": {826 "r2_heldout_text": -0.8259323728704582,827 "n_params": 3147264828 }829 },830 "collateral": {831 "antonym": {832 "condition": "ridge|L16|a1.0",833 "heldout_nll_base": 2.119346046447754,834 "heldout_nll_with_vector": 2.12896359761556,835 "cross_task": {836 "capital": {837 "zero": 0.0,838 "with_vector": 0.0839 },840 "morph": {841 "zero": 0.0,842 "with_vector": 0.0843 },844 "addthree": {845 "zero": 0.0,846 "with_vector": 0.0847 }848 }849 },850 "capital": {851 "condition": "ridge|L20|a1.0",852 "heldout_nll_base": 2.119346046447754,853 "heldout_nll_with_vector": 2.3801129976908366,854 "cross_task": {855 "antonym": {856 "zero": 0.0,857 "with_vector": 0.0858 },859 "morph": {860 "zero": 0.0,861 "with_vector": 0.0862 },863 "addthree": {864 "zero": 0.0,865 "with_vector": 0.0866 }867 }868 },869 "morph": {870 "condition": "procrustes|L2|a1.0",871 "heldout_nll_base": 2.119346046447754,872 "heldout_nll_with_vector": 2.1106110254923505,873 "cross_task": {874 "antonym": {875 "zero": 0.0,876 "with_vector": 0.0877 },878 "capital": {879 "zero": 0.0,880 "with_vector": 0.0881 },882 "addthree": {883 "zero": 0.0,884 "with_vector": 0.0885 }886 }887 },888 "addthree": {889 "condition": "procrustes|L2|a1.0",890 "heldout_nll_base": 2.119346046447754,891 "heldout_nll_with_vector": 2.104355494181315,892 "cross_task": {893 "antonym": {894 "zero": 0.0,895 "with_vector": 0.0896 },897 "capital": {898 "zero": 0.0,899 "with_vector": 0.0900 },901 "morph": {902 "zero": 0.0,903 "with_vector": 0.0904 }905 }906 }907 }908 }909 },910 "elapsed_seconds": 149.57248187065125,911 "run_id": "tier1_taskvector_quick_b89f92479155"912}