spb/localvm-research Public License
Running LLMs larger than memory on a consumer Mac — falsification-driven research: margin-gated deferred refinement, out-of-core verification on Apple Silicon. TR-01 published.
Python 63.2%
JavaScript 23.5%
CSS 11.8%
Shell 0.9%
Makefile 0.5%
1{2 "experiment": "expG_decision_stability",3 "author": "Simon-Pierre Boucher",4 "contact": "contact@spboucher.ai",5 "manifest": {6 "author": "Simon-Pierre Boucher",7 "contact": "contact@spboucher.ai",8 "project": "localvm-research",9 "collected_utc": "2026-08-12T05:11:29.643082+00:00",10 "chip": {11 "brand": "Apple M5 Max",12 "arch": "arm64",13 "cores_total": 18,14 "cores_performance": 6,15 "cores_efficiency": 12,16 "gpu_cores": 4017 },18 "memory": {19 "unified_bytes": 51539607552,20 "unified_gb": 48.0,21 "pagesize": 1638422 },23 "ssd": {24 "model": "APPLE SSD AP2048Z",25 "size": "2 TB",26 "smart_status": "Verified"27 },28 "os": {29 "product": "macOS",30 "version": "27.0",31 "build": "26A5388g",32 "kernel": "27.0.0"33 },34 "software": {35 "python": "3.14.4",36 "mlx": "0.32.0",37 "mlx_lm": "0.31.3",38 "torch": null,39 "numpy": "2.5.2"40 },41 "git": {42 "commit": "74574f3169ebc78c217e870cbb54b9696bfa0ac7",43 "dirty_tree": true44 },45 "thermal_level_at_collect": null46 },47 "config": {48 "model": "mlx-community/Qwen3-8B-bf16",49 "gen_tokens": 128,50 "bits": "3,4,8",51 "per_domain": 8,52 "group_size": 6453 },54 "n_trajectories": 48,55 "results_by_bits": {56 "3": {57 "bits": 3,58 "n_positions": 6144,59 "agreement_rate": 0.8527018229166666,60 "mean_kl_ref_q": 0.34334734082221985,61 "auroc_margin_predicts_disagreement": 0.883382071775749,62 "median_margin_agree": 7.5,63 "median_margin_disagree": 1.0,64 "escalation_curve": [65 {66 "tau": 0.0,67 "escalated_frac": 0.0,68 "residual_disagree": 0.1472981770833333469 },70 {71 "tau": 0.0,72 "escalated_frac": 0.0,73 "residual_disagree": 0.1472981770833333474 },75 {76 "tau": 0.0,77 "escalated_frac": 0.0,78 "residual_disagree": 0.1472981770833333479 },80 {81 "tau": 0.125,82 "escalated_frac": 0.01318359375,83 "residual_disagree": 0.1388346354166666684 },85 {86 "tau": 0.125,87 "escalated_frac": 0.01318359375,88 "residual_disagree": 0.1388346354166666689 },90 {91 "tau": 0.25,92 "escalated_frac": 0.021484375,93 "residual_disagree": 0.1326497395833333494 },95 {96 "tau": 0.25,97 "escalated_frac": 0.021484375,98 "residual_disagree": 0.1326497395833333499 },100 {101 "tau": 0.25,102 "escalated_frac": 0.021484375,103 "residual_disagree": 0.13264973958333334104 },105 {106 "tau": 0.25,107 "escalated_frac": 0.021484375,108 "residual_disagree": 0.13264973958333334109 },110 {111 "tau": 0.25,112 "escalated_frac": 0.021484375,113 "residual_disagree": 0.13264973958333334114 },115 {116 "tau": 0.375,117 "escalated_frac": 0.049967447916666664,118 "residual_disagree": 0.11865234375119 },120 {121 "tau": 0.375,122 "escalated_frac": 0.049967447916666664,123 "residual_disagree": 0.11865234375124 },125 {126 "tau": 0.5,127 "escalated_frac": 0.0576171875,128 "residual_disagree": 0.11474609375129 },130 {131 "tau": 0.5,132 "escalated_frac": 0.0576171875,133 "residual_disagree": 0.11474609375134 },135 {136 "tau": 0.5,137 "escalated_frac": 0.0576171875,138 "residual_disagree": 0.11474609375139 },140 {141 "tau": 0.5,142 "escalated_frac": 0.0576171875,143 "residual_disagree": 0.11474609375144 },145 {146 "tau": 0.5,147 "escalated_frac": 0.0576171875,148 "residual_disagree": 0.11474609375149 },150 {151 "tau": 0.5,152 "escalated_frac": 0.0576171875,153 "residual_disagree": 0.11474609375154 },155 {156 "tau": 0.625,157 "escalated_frac": 0.08821614583333333,158 "residual_disagree": 0.09977213541666667159 },160 {161 "tau": 0.75,162 "escalated_frac": 0.0927734375,163 "residual_disagree": 0.09781901041666667164 },165 {166 "tau": 0.75,167 "escalated_frac": 0.0927734375,168 "residual_disagree": 0.09781901041666667169 },170 {171 "tau": 0.75,172 "escalated_frac": 0.0927734375,173 "residual_disagree": 0.09781901041666667174 },175 {176 "tau": 0.75,177 "escalated_frac": 0.0927734375,178 "residual_disagree": 0.09781901041666667179 },180 {181 "tau": 0.8743718592964882,182 "escalated_frac": 0.11555989583333333,183 "residual_disagree": 0.08740234375184 },185 {186 "tau": 0.875,187 "escalated_frac": 0.11555989583333333,188 "residual_disagree": 0.08740234375189 },190 {191 "tau": 1.0,192 "escalated_frac": 0.12239583333333333,193 "residual_disagree": 0.0849609375194 },195 {196 "tau": 1.0,197 "escalated_frac": 0.12239583333333333,198 "residual_disagree": 0.0849609375199 },200 {201 "tau": 1.0,202 "escalated_frac": 0.12239583333333333,203 "residual_disagree": 0.0849609375204 },205 {206 "tau": 1.0,207 "escalated_frac": 0.12239583333333333,208 "residual_disagree": 0.0849609375209 },210 {211 "tau": 1.0,212 "escalated_frac": 0.12239583333333333,213 "residual_disagree": 0.0849609375214 },215 {216 "tau": 1.125,217 "escalated_frac": 0.14762369791666666,218 "residual_disagree": 0.0732421875219 },220 {221 "tau": 1.25,222 "escalated_frac": 0.15348307291666666,223 "residual_disagree": 0.07063802083333333224 },225 {226 "tau": 1.25,227 "escalated_frac": 0.15348307291666666,228 "residual_disagree": 0.07063802083333333229 },230 {231 "tau": 1.25,232 "escalated_frac": 0.15348307291666666,233 "residual_disagree": 0.07063802083333333234 },235 {236 "tau": 1.25,237 "escalated_frac": 0.15348307291666666,238 "residual_disagree": 0.07063802083333333239 },240 {241 "tau": 1.375,242 "escalated_frac": 0.1748046875,243 "residual_disagree": 0.0625244 },245 {246 "tau": 1.5,247 "escalated_frac": 0.18017578125,248 "residual_disagree": 0.060709635416666664249 },250 {251 "tau": 1.5,252 "escalated_frac": 0.18017578125,253 "residual_disagree": 0.060709635416666664254 },255 {256 "tau": 1.5,257 "escalated_frac": 0.18017578125,258 "residual_disagree": 0.060709635416666664259 },260 {261 "tau": 1.5,262 "escalated_frac": 0.18017578125,263 "residual_disagree": 0.060709635416666664264 },265 {266 "tau": 1.5,267 "escalated_frac": 0.18017578125,268 "residual_disagree": 0.060709635416666664269 },270 {271 "tau": 1.625,272 "escalated_frac": 0.20361328125,273 "residual_disagree": 0.0517578125274 },275 {276 "tau": 1.75,277 "escalated_frac": 0.20735677083333334,278 "residual_disagree": 0.049641927083333336279 },280 {281 "tau": 1.75,282 "escalated_frac": 0.20735677083333334,283 "residual_disagree": 0.049641927083333336284 },285 {286 "tau": 1.75,287 "escalated_frac": 0.20735677083333334,288 "residual_disagree": 0.049641927083333336289 },290 {291 "tau": 1.875,292 "escalated_frac": 0.22526041666666666,293 "residual_disagree": 0.043782552083333336294 },295 {296 "tau": 1.875,297 "escalated_frac": 0.22526041666666666,298 "residual_disagree": 0.043782552083333336299 },300 {301 "tau": 2.0,302 "escalated_frac": 0.23177083333333334,303 "residual_disagree": 0.041666666666666664304 },305 {306 "tau": 2.0,307 "escalated_frac": 0.23177083333333334,308 "residual_disagree": 0.041666666666666664309 },310 {311 "tau": 2.0,312 "escalated_frac": 0.23177083333333334,313 "residual_disagree": 0.041666666666666664314 },315 {316 "tau": 2.125,317 "escalated_frac": 0.24837239583333334,318 "residual_disagree": 0.037109375319 },320 {321 "tau": 2.25,322 "escalated_frac": 0.2535807291666667,323 "residual_disagree": 0.035807291666666664324 },325 {326 "tau": 2.25,327 "escalated_frac": 0.2535807291666667,328 "residual_disagree": 0.035807291666666664329 },330 {331 "tau": 2.25,332 "escalated_frac": 0.2535807291666667,333 "residual_disagree": 0.035807291666666664334 },335 {336 "tau": 2.375,337 "escalated_frac": 0.2706705729166667,338 "residual_disagree": 0.03076171875339 },340 {341 "tau": 2.375,342 "escalated_frac": 0.2706705729166667,343 "residual_disagree": 0.03076171875344 },345 {346 "tau": 2.5,347 "escalated_frac": 0.2766927083333333,348 "residual_disagree": 0.028971354166666668349 },350 {351 "tau": 2.5,352 "escalated_frac": 0.2766927083333333,353 "residual_disagree": 0.028971354166666668354 },355 {356 "tau": 2.5,357 "escalated_frac": 0.2766927083333333,358 "residual_disagree": 0.028971354166666668359 },360 {361 "tau": 2.625,362 "escalated_frac": 0.2955729166666667,363 "residual_disagree": 0.02490234375364 },365 {366 "tau": 2.75,367 "escalated_frac": 0.29931640625,368 "residual_disagree": 0.024251302083333332369 },370 {371 "tau": 2.75,372 "escalated_frac": 0.29931640625,373 "residual_disagree": 0.024251302083333332374 },375 {376 "tau": 2.75,377 "escalated_frac": 0.29931640625,378 "residual_disagree": 0.024251302083333332379 },380 {381 "tau": 2.75,382 "escalated_frac": 0.29931640625,383 "residual_disagree": 0.024251302083333332384 },385 {386 "tau": 3.0,387 "escalated_frac": 0.3214518229166667,388 "residual_disagree": 0.020182291666666668389 },390 {391 "tau": 3.0,392 "escalated_frac": 0.3214518229166667,393 "residual_disagree": 0.020182291666666668394 },395 {396 "tau": 3.0,397 "escalated_frac": 0.3214518229166667,398 "residual_disagree": 0.020182291666666668399 },400 {401 "tau": 3.0,402 "escalated_frac": 0.3214518229166667,403 "residual_disagree": 0.020182291666666668404 },405 {406 "tau": 3.125,407 "escalated_frac": 0.3390299479166667,408 "residual_disagree": 0.017740885416666668409 },410 {411 "tau": 3.25,412 "escalated_frac": 0.3426106770833333,413 "residual_disagree": 0.017252604166666668414 },415 {416 "tau": 3.25,417 "escalated_frac": 0.3426106770833333,418 "residual_disagree": 0.017252604166666668419 },420 {421 "tau": 3.25,422 "escalated_frac": 0.3426106770833333,423 "residual_disagree": 0.017252604166666668424 },425 {426 "tau": 3.4491206030151034,427 "escalated_frac": 0.36181640625,428 "residual_disagree": 0.01416015625429 },430 {431 "tau": 3.5,432 "escalated_frac": 0.36181640625,433 "residual_disagree": 0.01416015625434 },435 {436 "tau": 3.5,437 "escalated_frac": 0.36181640625,438 "residual_disagree": 0.01416015625439 },440 {441 "tau": 3.625,442 "escalated_frac": 0.3758138020833333,443 "residual_disagree": 0.012532552083333334444 },445 {446 "tau": 3.75,447 "escalated_frac": 0.3782552083333333,448 "residual_disagree": 0.01220703125449 },450 {451 "tau": 3.75,452 "escalated_frac": 0.3782552083333333,453 "residual_disagree": 0.01220703125454 },455 {456 "tau": 3.875,457 "escalated_frac": 0.3904622395833333,458 "residual_disagree": 0.0107421875459 },460 {461 "tau": 4.0,462 "escalated_frac": 0.39306640625,463 "residual_disagree": 0.010416666666666666464 },465 {466 "tau": 4.0,467 "escalated_frac": 0.39306640625,468 "residual_disagree": 0.010416666666666666469 },470 {471 "tau": 4.0,472 "escalated_frac": 0.39306640625,473 "residual_disagree": 0.010416666666666666474 },475 {476 "tau": 4.25,477 "escalated_frac": 0.4098307291666667,478 "residual_disagree": 0.00927734375479 },480 {481 "tau": 4.25,482 "escalated_frac": 0.4098307291666667,483 "residual_disagree": 0.00927734375484 },485 {486 "tau": 4.375,487 "escalated_frac": 0.4195963541666667,488 "residual_disagree": 0.008138020833333334489 },490 {491 "tau": 4.5,492 "escalated_frac": 0.4225260416666667,493 "residual_disagree": 0.007975260416666666494 },495 {496 "tau": 4.5,497 "escalated_frac": 0.4225260416666667,498 "residual_disagree": 0.007975260416666666499 },500 {501 "tau": 4.625,502 "escalated_frac": 0.4342447916666667,503 "residual_disagree": 0.006673177083333333504 },505 {506 "tau": 4.75,507 "escalated_frac": 0.4378255208333333,508 "residual_disagree": 0.006510416666666667509 },510 {511 "tau": 4.75,512 "escalated_frac": 0.4378255208333333,513 "residual_disagree": 0.006510416666666667514 },515 {516 "tau": 4.875,517 "escalated_frac": 0.4493815104166667,518 "residual_disagree": 0.005533854166666667519 },520 {521 "tau": 5.0,522 "escalated_frac": 0.45263671875,523 "residual_disagree": 0.00537109375524 },525 {526 "tau": 5.0,527 "escalated_frac": 0.45263671875,528 "residual_disagree": 0.00537109375529 },530 {531 "tau": 5.25,532 "escalated_frac": 0.4659830729166667,533 "residual_disagree": 0.005045572916666667534 },535 {536 "tau": 5.25,537 "escalated_frac": 0.4659830729166667,538 "residual_disagree": 0.005045572916666667539 },540 {541 "tau": 5.375,542 "escalated_frac": 0.4757486979166667,543 "residual_disagree": 0.00390625544 },545 {546 "tau": 5.5,547 "escalated_frac": 0.4788411458333333,548 "residual_disagree": 0.00390625549 },550 {551 "tau": 5.625,552 "escalated_frac": 0.4869791666666667,553 "residual_disagree": 0.0035807291666666665554 },555 {556 "tau": 5.75,557 "escalated_frac": 0.4900716145833333,558 "residual_disagree": 0.00341796875559 },560 {561 "tau": 5.75,562 "escalated_frac": 0.4900716145833333,563 "residual_disagree": 0.00341796875564 },565 {566 "tau": 6.0,567 "escalated_frac": 0.5009765625,568 "residual_disagree": 0.00341796875569 },570 {571 "tau": 6.0,572 "escalated_frac": 0.5009765625,573 "residual_disagree": 0.00341796875574 },575 {576 "tau": 6.125,577 "escalated_frac": 0.51171875,578 "residual_disagree": 0.0030924479166666665579 },580 {581 "tau": 6.25,582 "escalated_frac": 0.5152994791666666,583 "residual_disagree": 0.0029296875584 },585 {586 "tau": 6.25,587 "escalated_frac": 0.5152994791666666,588 "residual_disagree": 0.0029296875589 },590 {591 "tau": 6.5,592 "escalated_frac": 0.5242513020833334,593 "residual_disagree": 0.0026041666666666665594 },595 {596 "tau": 6.518844221105496,597 "escalated_frac": 0.53271484375,598 "residual_disagree": 0.0021158854166666665599 },600 {601 "tau": 6.75,602 "escalated_frac": 0.5349934895833334,603 "residual_disagree": 0.0021158854166666665604 },605 {606 "tau": 6.875,607 "escalated_frac": 0.5423177083333334,608 "residual_disagree": 0.001953125609 },610 {611 "tau": 7.0,612 "escalated_frac": 0.54443359375,613 "residual_disagree": 0.001953125614 },615 {616 "tau": 7.125,617 "escalated_frac": 0.5519205729166666,618 "residual_disagree": 0.001953125619 },620 {621 "tau": 7.25,622 "escalated_frac": 0.5545247395833334,623 "residual_disagree": 0.001953125624 },625 {626 "tau": 7.375,627 "escalated_frac": 0.5621744791666666,628 "residual_disagree": 0.0016276041666666667629 },630 {631 "tau": 7.5,632 "escalated_frac": 0.5647786458333334,633 "residual_disagree": 0.0016276041666666667634 },635 {636 "tau": 7.513190954773904,637 "escalated_frac": 0.5729166666666666,638 "residual_disagree": 0.00146484375639 },640 {641 "tau": 7.75,642 "escalated_frac": 0.5755208333333334,643 "residual_disagree": 0.00146484375644 },645 {646 "tau": 7.875,647 "escalated_frac": 0.5817057291666666,648 "residual_disagree": 0.0013020833333333333649 },650 {651 "tau": 8.0,652 "escalated_frac": 0.5841471354166666,653 "residual_disagree": 0.0013020833333333333654 },655 {656 "tau": 8.125,657 "escalated_frac": 0.59130859375,658 "residual_disagree": 0.0013020833333333333659 },660 {661 "tau": 8.25,662 "escalated_frac": 0.59423828125,663 "residual_disagree": 0.0013020833333333333664 },665 {666 "tau": 8.375,667 "escalated_frac": 0.6013997395833334,668 "residual_disagree": 0.0013020833333333333669 },670 {671 "tau": 8.5,672 "escalated_frac": 0.603515625,673 "residual_disagree": 0.0013020833333333333674 },675 {676 "tau": 8.625,677 "escalated_frac": 0.6101888020833334,678 "residual_disagree": 0.0013020833333333333679 },680 {681 "tau": 8.75,682 "escalated_frac": 0.61328125,683 "residual_disagree": 0.0013020833333333333684 },685 {686 "tau": 8.875,687 "escalated_frac": 0.6209309895833334,688 "residual_disagree": 0.0013020833333333333689 },690 {691 "tau": 9.0,692 "escalated_frac": 0.6238606770833334,693 "residual_disagree": 0.0013020833333333333694 },695 {696 "tau": 9.25,697 "escalated_frac": 0.6326497395833334,698 "residual_disagree": 0.0013020833333333333699 },700 {701 "tau": 9.25,702 "escalated_frac": 0.6326497395833334,703 "residual_disagree": 0.0013020833333333333704 },705 {706 "tau": 9.375,707 "escalated_frac": 0.6404622395833334,708 "residual_disagree": 0.0013020833333333333709 },710 {711 "tau": 9.5,712 "escalated_frac": 0.6437174479166666,713 "residual_disagree": 0.0013020833333333333714 },715 {716 "tau": 9.75,717 "escalated_frac": 0.6502278645833334,718 "residual_disagree": 0.0013020833333333333719 },720 {721 "tau": 9.875,722 "escalated_frac": 0.6560872395833334,723 "residual_disagree": 0.0011393229166666667724 },725 {726 "tau": 10.0,727 "escalated_frac": 0.65966796875,728 "residual_disagree": 0.0011393229166666667729 },730 {731 "tau": 10.25,732 "escalated_frac": 0.6669921875,733 "residual_disagree": 0.0009765625734 },735 {736 "tau": 10.375,737 "escalated_frac": 0.6715494791666666,738 "residual_disagree": 0.0009765625739 },740 {741 "tau": 10.5,742 "escalated_frac": 0.6741536458333334,743 "residual_disagree": 0.0009765625744 },745 {746 "tau": 10.75,747 "escalated_frac": 0.6818033854166666,748 "residual_disagree": 0.0009765625749 },750 {751 "tau": 10.875,752 "escalated_frac": 0.6865234375,753 "residual_disagree": 0.0009765625754 },755 {756 "tau": 11.1212311557789,757 "escalated_frac": 0.693359375,758 "residual_disagree": 0.0006510416666666666759 },760 {761 "tau": 11.25,762 "escalated_frac": 0.6941731770833334,763 "residual_disagree": 0.0006510416666666666764 },765 {766 "tau": 11.5,767 "escalated_frac": 0.7001953125,768 "residual_disagree": 0.0006510416666666666769 },770 {771 "tau": 11.75,772 "escalated_frac": 0.7080078125,773 "residual_disagree": 0.0006510416666666666774 },775 {776 "tau": 11.75,777 "escalated_frac": 0.7080078125,778 "residual_disagree": 0.0006510416666666666779 },780 {781 "tau": 12.0,782 "escalated_frac": 0.7154947916666666,783 "residual_disagree": 0.0006510416666666666784 },785 {786 "tau": 12.25,787 "escalated_frac": 0.7220052083333334,788 "residual_disagree": 0.00048828125789 },790 {791 "tau": 12.375,792 "escalated_frac": 0.7278645833333334,793 "residual_disagree": 0.00048828125794 },795 {796 "tau": 12.5,797 "escalated_frac": 0.7296549479166666,798 "residual_disagree": 0.00048828125799 },800 {801 "tau": 12.75,802 "escalated_frac": 0.736328125,803 "residual_disagree": 0.00048828125804 },805 {806 "tau": 13.0,807 "escalated_frac": 0.7405598958333334,808 "residual_disagree": 0.00048828125809 },810 {811 "tau": 13.25,812 "escalated_frac": 0.7467447916666666,813 "residual_disagree": 0.00048828125814 },815 {816 "tau": 13.5,817 "escalated_frac": 0.7517903645833334,818 "residual_disagree": 0.00048828125819 },820 {821 "tau": 13.75,822 "escalated_frac": 0.7576497395833334,823 "residual_disagree": 0.00048828125824 },825 {826 "tau": 13.875,827 "escalated_frac": 0.76171875,828 "residual_disagree": 0.0003255208333333333829 },830 {831 "tau": 14.001256281406995,832 "escalated_frac": 0.7688802083333334,833 "residual_disagree": 0.0003255208333333333834 },835 {836 "tau": 14.25,837 "escalated_frac": 0.7711588541666666,838 "residual_disagree": 0.0003255208333333333839 },840 {841 "tau": 14.5,842 "escalated_frac": 0.7760416666666666,843 "residual_disagree": 0.0003255208333333333844 },845 {846 "tau": 14.75,847 "escalated_frac": 0.7822265625,848 "residual_disagree": 0.0003255208333333333849 },850 {851 "tau": 15.0,852 "escalated_frac": 0.7884114583333334,853 "residual_disagree": 0.0003255208333333333854 },855 {856 "tau": 15.125,857 "escalated_frac": 0.7921549479166666,858 "residual_disagree": 0.0003255208333333333859 },860 {861 "tau": 15.375,862 "escalated_frac": 0.7981770833333334,863 "residual_disagree": 0.00016276041666666666864 },865 {866 "tau": 15.5,867 "escalated_frac": 0.7999674479166666,868 "residual_disagree": 0.00016276041666666666869 },870 {871 "tau": 15.75,872 "escalated_frac": 0.8063151041666666,873 "residual_disagree": 0.00016276041666666666874 },875 {876 "tau": 16.0,877 "escalated_frac": 0.81201171875,878 "residual_disagree": 0.0879 },880 {881 "tau": 16.25,882 "escalated_frac": 0.8172200520833334,883 "residual_disagree": 0.0884 },885 {886 "tau": 16.5,887 "escalated_frac": 0.82177734375,888 "residual_disagree": 0.0889 },890 {891 "tau": 16.75,892 "escalated_frac": 0.8279622395833334,893 "residual_disagree": 0.0894 },895 {896 "tau": 17.0,897 "escalated_frac": 0.8317057291666666,898 "residual_disagree": 0.0899 },900 {901 "tau": 17.25,902 "escalated_frac": 0.8365885416666666,903 "residual_disagree": 0.0904 },905 {906 "tau": 17.5,907 "escalated_frac": 0.84326171875,908 "residual_disagree": 0.0909 },910 {911 "tau": 17.75,912 "escalated_frac": 0.8473307291666666,913 "residual_disagree": 0.0914 },915 {916 "tau": 18.0,917 "escalated_frac": 0.8522135416666666,918 "residual_disagree": 0.0919 },920 {921 "tau": 18.25,922 "escalated_frac": 0.857421875,923 "residual_disagree": 0.0924 },925 {926 "tau": 18.5,927 "escalated_frac": 0.86279296875,928 "residual_disagree": 0.0929 },930 {931 "tau": 18.75,932 "escalated_frac": 0.8678385416666666,933 "residual_disagree": 0.0934 },935 {936 "tau": 19.0,937 "escalated_frac": 0.87255859375,938 "residual_disagree": 0.0939 },940 {941 "tau": 19.25,942 "escalated_frac": 0.8780924479166666,943 "residual_disagree": 0.0944 },945 {946 "tau": 19.37562814070361,947 "escalated_frac": 0.8844401041666666,948 "residual_disagree": 0.0949 },950 {951 "tau": 19.75,952 "escalated_frac": 0.8880208333333334,953 "residual_disagree": 0.0954 },955 {956 "tau": 19.9679648241206,957 "escalated_frac": 0.8943684895833334,958 "residual_disagree": 0.0959 },960 {961 "tau": 20.25,962 "escalated_frac": 0.8984375,963 "residual_disagree": 0.0964 },965 {966 "tau": 20.5,967 "escalated_frac": 0.9020182291666666,968 "residual_disagree": 0.0969 },970 {971 "tau": 20.75,972 "escalated_frac": 0.90771484375,973 "residual_disagree": 0.0974 },975 {976 "tau": 21.125,977 "escalated_frac": 0.9140625,978 "residual_disagree": 0.0979 },980 {981 "tau": 21.375,982 "escalated_frac": 0.9178059895833334,983 "residual_disagree": 0.0984 },985 {986 "tau": 21.744974874371906,987 "escalated_frac": 0.9244791666666666,988 "residual_disagree": 0.0989 },990 {991 "tau": 22.0,992 "escalated_frac": 0.9287109375,993 "residual_disagree": 0.0994 },995 {996 "tau": 22.462311557789008,997 "escalated_frac": 0.9345703125,998 "residual_disagree": 0.0999 },1000 {1001 "tau": 22.75,1002 "escalated_frac": 0.93798828125,1003 "residual_disagree": 0.01004 },1005 {1006 "tau": 23.25,1007 "escalated_frac": 0.9443359375,1008 "residual_disagree": 0.01009 },1010 {1011 "tau": 23.625,1012 "escalated_frac": 0.9482421875,1013 "residual_disagree": 0.01014 },1015 {1016 "tau": 24.021984924623098,1017 "escalated_frac": 0.9547526041666666,1018 "residual_disagree": 0.01019 },1020 {1021 "tau": 24.5,1022 "escalated_frac": 0.9581705729166666,1023 "residual_disagree": 0.01024 },1025 {1026 "tau": 25.125,1027 "escalated_frac": 0.9640299479166666,1028 "residual_disagree": 0.01029 },1030 {1031 "tau": 25.722989949748694,1032 "escalated_frac": 0.9697265625,1033 "residual_disagree": 0.01034 },1035 {1036 "tau": 26.375,1037 "escalated_frac": 0.974609375,1038 "residual_disagree": 0.01039 },1040 {1041 "tau": 27.125,1042 "escalated_frac": 0.9794921875,1043 "residual_disagree": 0.01044 },1045 {1046 "tau": 28.25,1047 "escalated_frac": 0.984375,1048 "residual_disagree": 0.01049 },1050 {1051 "tau": 29.28266331658301,1052 "escalated_frac": 0.9899088541666666,1053 "residual_disagree": 0.01054 },1055 {1056 "tau": 31.25,1057 "escalated_frac": 0.99462890625,1058 "residual_disagree": 0.01059 },1060 {1061 "tau": 40.0,1062 "escalated_frac": 0.9998372395833334,1063 "residual_disagree": 0.01064 }1065 ],1066 "per_domain": {1067 "code": {1068 "agreement_rate": 0.8603515625,1069 "auroc": 0.88244445679178941070 },1071 "math": {1072 "agreement_rate": 0.90234375,1073 "auroc": 0.90818181818181821074 },1075 "chat": {1076 "agreement_rate": 0.8173828125,1077 "auroc": 0.86694267149675121078 },1079 "french": {1080 "agreement_rate": 0.8359375,1081 "auroc": 0.88213451268357811082 },1083 "english": {1084 "agreement_rate": 0.822265625,1085 "auroc": 0.87015478583174541086 },1087 "reasoning": {1088 "agreement_rate": 0.8779296875,1089 "auroc": 0.89319688542825361090 }1091 },1092 "escalation_frac_for_99pct": 0.40983072916666671093 },1094 "4": {1095 "bits": 4,1096 "n_positions": 6144,1097 "agreement_rate": 0.9212239583333334,1098 "mean_kl_ref_q": 0.08975151926279068,1099 "auroc_margin_predicts_disagreement": 0.9199168443185468,1100 "median_margin_agree": 7.125,1101 "median_margin_disagree": 0.5,1102 "escalation_curve": [1103 {1104 "tau": 0.0,1105 "escalated_frac": 0.0,1106 "residual_disagree": 0.078776041666666671107 },1108 {1109 "tau": 0.0,1110 "escalated_frac": 0.0,1111 "residual_disagree": 0.078776041666666671112 },1113 {1114 "tau": 0.0,1115 "escalated_frac": 0.0,1116 "residual_disagree": 0.078776041666666671117 },1118 {1119 "tau": 0.125,1120 "escalated_frac": 0.013834635416666666,1121 "residual_disagree": 0.071777343751122 },1123 {1124 "tau": 0.125,1125 "escalated_frac": 0.013834635416666666,1126 "residual_disagree": 0.071777343751127 },1128 {1129 "tau": 0.25,1130 "escalated_frac": 0.0205078125,1131 "residual_disagree": 0.0683593751132 },1133 {1134 "tau": 0.25,1135 "escalated_frac": 0.0205078125,1136 "residual_disagree": 0.0683593751137 },1138 {1139 "tau": 0.25,1140 "escalated_frac": 0.0205078125,1141 "residual_disagree": 0.0683593751142 },1143 {1144 "tau": 0.25,1145 "escalated_frac": 0.0205078125,1146 "residual_disagree": 0.0683593751147 },1148 {1149 "tau": 0.25,1150 "escalated_frac": 0.0205078125,1151 "residual_disagree": 0.0683593751152 },1153 {1154 "tau": 0.25,1155 "escalated_frac": 0.0205078125,1156 "residual_disagree": 0.0683593751157 },1158 {1159 "tau": 0.375,1160 "escalated_frac": 0.053059895833333336,1161 "residual_disagree": 0.053222656251162 },1163 {1164 "tau": 0.5,1165 "escalated_frac": 0.059244791666666664,1166 "residual_disagree": 0.050292968751167 },1168 {1169 "tau": 0.5,1170 "escalated_frac": 0.059244791666666664,1171 "residual_disagree": 0.050292968751172 },1173 {1174 "tau": 0.5,1175 "escalated_frac": 0.059244791666666664,1176 "residual_disagree": 0.050292968751177 },1178 {1179 "tau": 0.5,1180 "escalated_frac": 0.059244791666666664,1181 "residual_disagree": 0.050292968751182 },1183 {1184 "tau": 0.5,1185 "escalated_frac": 0.059244791666666664,1186 "residual_disagree": 0.050292968751187 },1188 {1189 "tau": 0.5,1190 "escalated_frac": 0.059244791666666664,1191 "residual_disagree": 0.050292968751192 },1193 {1194 "tau": 0.625,1195 "escalated_frac": 0.08870442708333333,1196 "residual_disagree": 0.0388997395833333361197 },1198 {1199 "tau": 0.75,1200 "escalated_frac": 0.09440104166666667,1201 "residual_disagree": 0.0372721354166666641202 },1203 {1204 "tau": 0.75,1205 "escalated_frac": 0.09440104166666667,1206 "residual_disagree": 0.0372721354166666641207 },1208 {1209 "tau": 0.75,1210 "escalated_frac": 0.09440104166666667,1211 "residual_disagree": 0.0372721354166666641212 },1213 {1214 "tau": 0.75,1215 "escalated_frac": 0.09440104166666667,1216 "residual_disagree": 0.0372721354166666641217 },1218 {1219 "tau": 0.75,1220 "escalated_frac": 0.09440104166666667,1221 "residual_disagree": 0.0372721354166666641222 },1223 {1224 "tau": 0.875,1225 "escalated_frac": 0.11735026041666667,1226 "residual_disagree": 0.0292968751227 },1228 {1229 "tau": 1.0,1230 "escalated_frac": 0.12386067708333333,1231 "residual_disagree": 0.026855468751232 },1233 {1234 "tau": 1.0,1235 "escalated_frac": 0.12386067708333333,1236 "residual_disagree": 0.026855468751237 },1238 {1239 "tau": 1.0,1240 "escalated_frac": 0.12386067708333333,1241 "residual_disagree": 0.026855468751242 },1243 {1244 "tau": 1.0,1245 "escalated_frac": 0.12386067708333333,1246 "residual_disagree": 0.026855468751247 },1248 {1249 "tau": 1.0,1250 "escalated_frac": 0.12386067708333333,1251 "residual_disagree": 0.026855468751252 },1253 {1254 "tau": 1.125,1255 "escalated_frac": 0.14680989583333334,1256 "residual_disagree": 0.021972656251257 },1258 {1259 "tau": 1.25,1260 "escalated_frac": 0.15185546875,1261 "residual_disagree": 0.0206705729166666681262 },1263 {1264 "tau": 1.25,1265 "escalated_frac": 0.15185546875,1266 "residual_disagree": 0.0206705729166666681267 },1268 {1269 "tau": 1.25,1270 "escalated_frac": 0.15185546875,1271 "residual_disagree": 0.0206705729166666681272 },1273 {1274 "tau": 1.25,1275 "escalated_frac": 0.15185546875,1276 "residual_disagree": 0.0206705729166666681277 },1278 {1279 "tau": 1.375,1280 "escalated_frac": 0.17366536458333334,1281 "residual_disagree": 0.0167643229166666681282 },1283 {1284 "tau": 1.5,1285 "escalated_frac": 0.17708333333333334,1286 "residual_disagree": 0.0162760416666666681287 },1288 {1289 "tau": 1.5,1290 "escalated_frac": 0.17708333333333334,1291 "residual_disagree": 0.0162760416666666681292 },1293 {1294 "tau": 1.5,1295 "escalated_frac": 0.17708333333333334,1296 "residual_disagree": 0.0162760416666666681297 },1298 {1299 "tau": 1.5,1300 "escalated_frac": 0.17708333333333334,1301 "residual_disagree": 0.0162760416666666681302 },1303 {1304 "tau": 1.625,1305 "escalated_frac": 0.197265625,1306 "residual_disagree": 0.0128580729166666661307 },1308 {1309 "tau": 1.75,1310 "escalated_frac": 0.201171875,1311 "residual_disagree": 0.012207031251312 },1313 {1314 "tau": 1.75,1315 "escalated_frac": 0.201171875,1316 "residual_disagree": 0.012207031251317 },1318 {1319 "tau": 1.75,1320 "escalated_frac": 0.201171875,1321 "residual_disagree": 0.012207031251322 },1323 {1324 "tau": 1.75,1325 "escalated_frac": 0.201171875,1326 "residual_disagree": 0.012207031251327 },1328 {1329 "tau": 2.0,1330 "escalated_frac": 0.22607421875,1331 "residual_disagree": 0.0099283854166666661332 },1333 {1334 "tau": 2.0,1335 "escalated_frac": 0.22607421875,1336 "residual_disagree": 0.0099283854166666661337 },1338 {1339 "tau": 2.0,1340 "escalated_frac": 0.22607421875,1341 "residual_disagree": 0.0099283854166666661342 },1343 {1344 "tau": 2.0,1345 "escalated_frac": 0.22607421875,1346 "residual_disagree": 0.0099283854166666661347 },1348 {1349 "tau": 2.125,1350 "escalated_frac": 0.24527994791666666,1351 "residual_disagree": 0.0086263020833333341352 },1353 {1354 "tau": 2.25,1355 "escalated_frac": 0.25,1356 "residual_disagree": 0.0084635416666666661357 },1358 {1359 "tau": 2.25,1360 "escalated_frac": 0.25,1361 "residual_disagree": 0.0084635416666666661362 },1363 {1364 "tau": 2.25,1365 "escalated_frac": 0.25,1366 "residual_disagree": 0.0084635416666666661367 },1368 {1369 "tau": 2.25,1370 "escalated_frac": 0.25,1371 "residual_disagree": 0.0084635416666666661372 },1373 {1374 "tau": 2.375,1375 "escalated_frac": 0.26953125,1376 "residual_disagree": 0.006347656251377 },1378 {1379 "tau": 2.5,1380 "escalated_frac": 0.275390625,1381 "residual_disagree": 0.0061848958333333331382 },1383 {1384 "tau": 2.5,1385 "escalated_frac": 0.275390625,1386 "residual_disagree": 0.0061848958333333331387 },1388 {1389 "tau": 2.5,1390 "escalated_frac": 0.275390625,1391 "residual_disagree": 0.0061848958333333331392 },1393 {1394 "tau": 2.625,1395 "escalated_frac": 0.291015625,1396 "residual_disagree": 0.005371093751397 },1398 {1399 "tau": 2.75,1400 "escalated_frac": 0.2950846354166667,1401 "residual_disagree": 0.0052083333333333331402 },1403 {1404 "tau": 2.75,1405 "escalated_frac": 0.2950846354166667,1406 "residual_disagree": 0.0052083333333333331407 },1408 {1409 "tau": 2.75,1410 "escalated_frac": 0.2950846354166667,1411 "residual_disagree": 0.0052083333333333331412 },1413 {1414 "tau": 2.75,1415 "escalated_frac": 0.2950846354166667,1416 "residual_disagree": 0.0052083333333333331417 },1418 {1419 "tau": 3.0,1420 "escalated_frac": 0.3147786458333333,1421 "residual_disagree": 0.003906251422 },1423 {1424 "tau": 3.0,1425 "escalated_frac": 0.3147786458333333,1426 "residual_disagree": 0.003906251427 },1428 {1429 "tau": 3.0,1430 "escalated_frac": 0.3147786458333333,1431 "residual_disagree": 0.003906251432 },1433 {1434 "tau": 3.125,1435 "escalated_frac": 0.3308919270833333,1436 "residual_disagree": 0.00309244791666666651437 },1438 {1439 "tau": 3.25,1440 "escalated_frac": 0.3356119791666667,1441 "residual_disagree": 0.00309244791666666651442 },1443 {1444 "tau": 3.25,1445 "escalated_frac": 0.3356119791666667,1446 "residual_disagree": 0.00309244791666666651447 },1448 {1449 "tau": 3.25,1450 "escalated_frac": 0.3356119791666667,1451 "residual_disagree": 0.00309244791666666651452 },1453 {1454 "tau": 3.375,1455 "escalated_frac": 0.3505859375,1456 "residual_disagree": 0.00227864583333333351457 },1458 {1459 "tau": 3.5,1460 "escalated_frac": 0.3541666666666667,1461 "residual_disagree": 0.00211588541666666651462 },1463 {1464 "tau": 3.5,1465 "escalated_frac": 0.3541666666666667,1466 "residual_disagree": 0.00211588541666666651467 },1468 {1469 "tau": 3.5,1470 "escalated_frac": 0.3541666666666667,1471 "residual_disagree": 0.00211588541666666651472 },1473 {1474 "tau": 3.625,1475 "escalated_frac": 0.3693033854166667,1476 "residual_disagree": 0.00162760416666666671477 },1478 {1479 "tau": 3.75,1480 "escalated_frac": 0.3727213541666667,1481 "residual_disagree": 0.00162760416666666671482 },1483 {1484 "tau": 3.75,1485 "escalated_frac": 0.3727213541666667,1486 "residual_disagree": 0.00162760416666666671487 },1488 {1489 "tau": 3.875,1490 "escalated_frac": 0.3841145833333333,1491 "residual_disagree": 0.001464843751492 },1493 {1494 "tau": 4.0,1495 "escalated_frac": 0.3875325520833333,1496 "residual_disagree": 0.001464843751497 },1498 {1499 "tau": 4.0,1500 "escalated_frac": 0.3875325520833333,1501 "residual_disagree": 0.001464843751502 },1503 {1504 "tau": 4.25,1505 "escalated_frac": 0.4007161458333333,1506 "residual_disagree": 0.00097656251507 },1508 {1509 "tau": 4.25,1510 "escalated_frac": 0.4007161458333333,1511 "residual_disagree": 0.00097656251512 },1513 {1514 "tau": 4.25,1515 "escalated_frac": 0.4007161458333333,1516 "residual_disagree": 0.00097656251517 },1518 {1519 "tau": 4.5,1520 "escalated_frac": 0.416015625,1521 "residual_disagree": 0.00097656251522 },1523 {1524 "tau": 4.5,1525 "escalated_frac": 0.416015625,1526 "residual_disagree": 0.00097656251527 },1528 {1529 "tau": 4.5,1530 "escalated_frac": 0.416015625,1531 "residual_disagree": 0.00097656251532 },1533 {1534 "tau": 4.625,1535 "escalated_frac": 0.4305013020833333,1536 "residual_disagree": 0.00065104166666666661537 },1538 {1539 "tau": 4.75,1540 "escalated_frac": 0.43310546875,1541 "residual_disagree": 0.00065104166666666661542 },1543 {1544 "tau": 4.75,1545 "escalated_frac": 0.43310546875,1546 "residual_disagree": 0.00065104166666666661547 },1548 {1549 "tau": 5.0,1550 "escalated_frac": 0.4449869791666667,1551 "residual_disagree": 0.00065104166666666661552 },1553 {1554 "tau": 5.0,1555 "escalated_frac": 0.4449869791666667,1556 "residual_disagree": 0.00065104166666666661557 },1558 {1559 "tau": 5.25,1560 "escalated_frac": 0.4568684895833333,1561 "residual_disagree": 0.000488281251562 },1563 {1564 "tau": 5.25,1565 "escalated_frac": 0.4568684895833333,1566 "residual_disagree": 0.000488281251567 },1568 {1569 "tau": 5.375,1570 "escalated_frac": 0.4664713541666667,1571 "residual_disagree": 0.000488281251572 },1573 {1574 "tau": 5.5,1575 "escalated_frac": 0.46826171875,1576 "residual_disagree": 0.000488281251577 },1578 {1579 "tau": 5.5,1580 "escalated_frac": 0.46826171875,1581 "residual_disagree": 0.000488281251582 },1583 {1584 "tau": 5.75,1585 "escalated_frac": 0.48046875,1586 "residual_disagree": 0.000488281251587 },1588 {1589 "tau": 5.75,1590 "escalated_frac": 0.48046875,1591 "residual_disagree": 0.000488281251592 },1593 {1594 "tau": 6.0,1595 "escalated_frac": 0.4910481770833333,1596 "residual_disagree": 0.00032552083333333331597 },1598 {1599 "tau": 6.0,1600 "escalated_frac": 0.4910481770833333,1601 "residual_disagree": 0.00032552083333333331602 },1603 {1604 "tau": 6.125,1605 "escalated_frac": 0.5009765625,1606 "residual_disagree": 0.00032552083333333331607 },1608 {1609 "tau": 6.25,1610 "escalated_frac": 0.5026041666666666,1611 "residual_disagree": 0.00032552083333333331612 },1613 {1614 "tau": 6.375,1615 "escalated_frac": 0.51025390625,1616 "residual_disagree": 0.00032552083333333331617 },1618 {1619 "tau": 6.5,1620 "escalated_frac": 0.5130208333333334,1621 "residual_disagree": 0.00032552083333333331622 },1623 {1624 "tau": 6.625,1625 "escalated_frac": 0.52099609375,1626 "residual_disagree": 0.00032552083333333331627 },1628 {1629 "tau": 6.75,1630 "escalated_frac": 0.5234375,1631 "residual_disagree": 0.00032552083333333331632 },1633 {1634 "tau": 7.0,1635 "escalated_frac": 0.5305989583333334,1636 "residual_disagree": 0.00032552083333333331637 },1638 {1639 "tau": 7.0,1640 "escalated_frac": 0.5305989583333334,1641 "residual_disagree": 0.00032552083333333331642 },1643 {1644 "tau": 7.25,1645 "escalated_frac": 0.541015625,1646 "residual_disagree": 0.000162760416666666661647 },1648 {1649 "tau": 7.25,1650 "escalated_frac": 0.541015625,1651 "residual_disagree": 0.000162760416666666661652 },1653 {1654 "tau": 7.5,1655 "escalated_frac": 0.55029296875,1656 "residual_disagree": 0.000162760416666666661657 },1658 {1659 "tau": 7.625,1660 "escalated_frac": 0.5568033854166666,1661 "residual_disagree": 0.000162760416666666661662 },1663 {1664 "tau": 7.75,1665 "escalated_frac": 0.5589192708333334,1666 "residual_disagree": 0.000162760416666666661667 },1668 {1669 "tau": 7.875,1670 "escalated_frac": 0.5673828125,1671 "residual_disagree": 0.000162760416666666661672 },1673 {1674 "tau": 8.0,1675 "escalated_frac": 0.5690104166666666,1676 "residual_disagree": 0.000162760416666666661677 },1678 {1679 "tau": 8.25,1680 "escalated_frac": 0.5763346354166666,1681 "residual_disagree": 0.000162760416666666661682 },1683 {1684 "tau": 8.375,1685 "escalated_frac": 0.58203125,1686 "residual_disagree": 0.000162760416666666661687 },1688 {1689 "tau": 8.5,1690 "escalated_frac": 0.58349609375,1691 "residual_disagree": 0.000162760416666666661692 },1693 {1694 "tau": 8.75,1695 "escalated_frac": 0.5908203125,1696 "residual_disagree": 0.000162760416666666661697 },1698 {1699 "tau": 8.875,1700 "escalated_frac": 0.595703125,1701 "residual_disagree": 0.000162760416666666661702 },1703 {1704 "tau": 9.0,1705 "escalated_frac": 0.5983072916666666,1706 "residual_disagree": 0.000162760416666666661707 },1708 {1709 "tau": 9.25,1710 "escalated_frac": 0.6064453125,1711 "residual_disagree": 0.000162760416666666661712 },1713 {1714 "tau": 9.375,1715 "escalated_frac": 0.6111653645833334,1716 "residual_disagree": 0.000162760416666666661717 },1718 {1719 "tau": 9.5,1720 "escalated_frac": 0.6139322916666666,1721 "residual_disagree": 0.000162760416666666661722 },1723 {1724 "tau": 9.75,1725 "escalated_frac": 0.62060546875,1726 "residual_disagree": 0.000162760416666666661727 },1728 {1729 "tau": 10.0,1730 "escalated_frac": 0.6271158854166666,1731 "residual_disagree": 0.01732 },1733 {1734 "tau": 10.0,1735 "escalated_frac": 0.6271158854166666,1736 "residual_disagree": 0.01737 },1738 {1739 "tau": 10.25,1740 "escalated_frac": 0.6355794270833334,1741 "residual_disagree": 0.01742 },1743 {1744 "tau": 10.5,1745 "escalated_frac": 0.64306640625,1746 "residual_disagree": 0.01747 },1748 {1749 "tau": 10.5,1750 "escalated_frac": 0.64306640625,1751 "residual_disagree": 0.01752 },1753 {1754 "tau": 10.75,1755 "escalated_frac": 0.6510416666666666,1756 "residual_disagree": 0.01757 },1758 {1759 "tau": 11.0,1760 "escalated_frac": 0.6564127604166666,1761 "residual_disagree": 0.01762 },1763 {1764 "tau": 11.25,1765 "escalated_frac": 0.66259765625,1766 "residual_disagree": 0.01767 },1768 {1769 "tau": 11.375,1770 "escalated_frac": 0.6673177083333334,1771 "residual_disagree": 0.01772 },1773 {1774 "tau": 11.5,1775 "escalated_frac": 0.6695963541666666,1776 "residual_disagree": 0.01777 },1778 {1779 "tau": 11.75,1780 "escalated_frac": 0.6754557291666666,1781 "residual_disagree": 0.01782 },1783 {1784 "tau": 12.0,1785 "escalated_frac": 0.6814778645833334,1786 "residual_disagree": 0.01787 },1788 {1789 "tau": 12.25,1790 "escalated_frac": 0.6871744791666666,1791 "residual_disagree": 0.01792 },1793 {1794 "tau": 12.25,1795 "escalated_frac": 0.6871744791666666,1796 "residual_disagree": 0.01797 },1798 {1799 "tau": 12.5,1800 "escalated_frac": 0.6956380208333334,1801 "residual_disagree": 0.01802 },1803 {1804 "tau": 12.75,1805 "escalated_frac": 0.7021484375,1806 "residual_disagree": 0.01807 },1808 {1809 "tau": 12.875,1810 "escalated_frac": 0.70703125,1811 "residual_disagree": 0.01812 },1813 {1814 "tau": 13.125,1815 "escalated_frac": 0.7130533854166666,1816 "residual_disagree": 0.01817 },1818 {1819 "tau": 13.25,1820 "escalated_frac": 0.7137044270833334,1821 "residual_disagree": 0.01822 },1823 {1824 "tau": 13.5,1825 "escalated_frac": 0.7210286458333334,1826 "residual_disagree": 0.01827 },1828 {1829 "tau": 13.75,1830 "escalated_frac": 0.7275390625,1831 "residual_disagree": 0.01832 },1833 {1834 "tau": 14.0,1835 "escalated_frac": 0.7327473958333334,1836 "residual_disagree": 0.01837 },1838 {1839 "tau": 14.125,1840 "escalated_frac": 0.73779296875,1841 "residual_disagree": 0.01842 },1843 {1844 "tau": 14.375,1845 "escalated_frac": 0.7425130208333334,1846 "residual_disagree": 0.01847 },1848 {1849 "tau": 14.5,1850 "escalated_frac": 0.744140625,1851 "residual_disagree": 0.01852 },1853 {1854 "tau": 14.75,1855 "escalated_frac": 0.75146484375,1856 "residual_disagree": 0.01857 },1858 {1859 "tau": 15.0,1860 "escalated_frac": 0.75830078125,1861 "residual_disagree": 0.01862 },1863 {1864 "tau": 15.125,1865 "escalated_frac": 0.7618815104166666,1866 "residual_disagree": 0.01867 },1868 {1869 "tau": 15.375,1870 "escalated_frac": 0.7682291666666666,1871 "residual_disagree": 0.01872 },1873 {1874 "tau": 15.75,1875 "escalated_frac": 0.7734375,1876 "residual_disagree": 0.01877 },1878 {1879 "tau": 15.875,1880 "escalated_frac": 0.7766927083333334,1881 "residual_disagree": 0.01882 },1883 {1884 "tau": 16.0,1885 "escalated_frac": 0.77978515625,1886 "residual_disagree": 0.01887 },1888 {1889 "tau": 16.25,1890 "escalated_frac": 0.7859700520833334,1891 "residual_disagree": 0.01892 },1893 {1894 "tau": 16.625,1895 "escalated_frac": 0.7931315104166666,1896 "residual_disagree": 0.01897 },1898 {1899 "tau": 16.875,1900 "escalated_frac": 0.7980143229166666,1901 "residual_disagree": 0.01902 },1903 {1904 "tau": 17.0,1905 "escalated_frac": 0.8004557291666666,1906 "residual_disagree": 0.01907 },1908 {1909 "tau": 17.375,1910 "escalated_frac": 0.8084309895833334,1911 "residual_disagree": 0.01912 },1913 {1914 "tau": 17.625,1915 "escalated_frac": 0.8126627604166666,1916 "residual_disagree": 0.01917 },1918 {1919 "tau": 17.875,1920 "escalated_frac": 0.81787109375,1921 "residual_disagree": 0.01922 },1923 {1924 "tau": 18.125,1925 "escalated_frac": 0.8232421875,1926 "residual_disagree": 0.01927 },1928 {1929 "tau": 18.375,1930 "escalated_frac": 0.8284505208333334,1931 "residual_disagree": 0.01932 },1933 {1934 "tau": 18.625,1935 "escalated_frac": 0.8330078125,1936 "residual_disagree": 0.01937 },1938 {1939 "tau": 18.75,1940 "escalated_frac": 0.83544921875,1941 "residual_disagree": 0.01942 },1943 {1944 "tau": 19.0,1945 "escalated_frac": 0.8421223958333334,1946 "residual_disagree": 0.01947 },1948 {1949 "tau": 19.25,1950 "escalated_frac": 0.8479817708333334,1951 "residual_disagree": 0.01952 },1953 {1954 "tau": 19.375,1955 "escalated_frac": 0.8523763020833334,1956 "residual_disagree": 0.01957 },1958 {1959 "tau": 19.625,1960 "escalated_frac": 0.8577473958333334,1961 "residual_disagree": 0.01962 },1963 {1964 "tau": 20.0,1965 "escalated_frac": 0.8631184895833334,1966 "residual_disagree": 0.01967 },1968 {1969 "tau": 20.25,1970 "escalated_frac": 0.8689778645833334,1971 "residual_disagree": 0.01972 },1973 {1974 "tau": 20.375,1975 "escalated_frac": 0.8727213541666666,1976 "residual_disagree": 0.01977 },1978 {1979 "tau": 20.625,1980 "escalated_frac": 0.8790690104166666,1981 "residual_disagree": 0.01982 },1983 {1984 "tau": 21.0,1985 "escalated_frac": 0.88427734375,1986 "residual_disagree": 0.01987 },1988 {1989 "tau": 21.125,1990 "escalated_frac": 0.8885091145833334,1991 "residual_disagree": 0.01992 },1993 {1994 "tau": 21.4839824120603,1995 "escalated_frac": 0.8943684895833334,1996 "residual_disagree": 0.01997 },1998 {1999 "tau": 21.75,2000 "escalated_frac": 0.8987630208333334,2001 "residual_disagree": 0.02002 },2003 {2004 "tau": 22.0,2005 "escalated_frac": 0.9015299479166666,2006 "residual_disagree": 0.02007 },2008 {2009 "tau": 22.5,2010 "escalated_frac": 0.9083658854166666,2011 "residual_disagree": 0.02012 },2013 {2014 "tau": 22.77763819095469,2015 "escalated_frac": 0.91455078125,2016 "residual_disagree": 0.02017 },2018 {2019 "tau": 23.25,2020 "escalated_frac": 0.91943359375,2021 "residual_disagree": 0.02022 },2023 {2024 "tau": 23.5,2025 "escalated_frac": 0.9226888020833334,2026 "residual_disagree": 0.02027 },2028 {2029 "tau": 23.875,2030 "escalated_frac": 0.9288736979166666,2031 "residual_disagree": 0.02032 },2033 {2034 "tau": 24.25,2035 "escalated_frac": 0.9329427083333334,2036 "residual_disagree": 0.02037 },2038 {2039 "tau": 24.5,2040 "escalated_frac": 0.9366861979166666,2041 "residual_disagree": 0.02042 },2043 {2044 "tau": 25.0,2045 "escalated_frac": 0.9441731770833334,2046 "residual_disagree": 0.02047 },2048 {2049 "tau": 25.375,2050 "escalated_frac": 0.9493815104166666,2051 "residual_disagree": 0.02052 },2053 {2054 "tau": 25.75,2055 "escalated_frac": 0.9534505208333334,2056 "residual_disagree": 0.02057 },2058 {2059 "tau": 26.25,2060 "escalated_frac": 0.9581705729166666,2061 "residual_disagree": 0.02062 },2063 {2064 "tau": 26.875,2065 "escalated_frac": 0.96435546875,2066 "residual_disagree": 0.02067 },2068 {2069 "tau": 27.472989949748694,2070 "escalated_frac": 0.9697265625,2071 "residual_disagree": 0.02072 },2073 {2074 "tau": 28.25,2075 "escalated_frac": 0.9742838541666666,2076 "residual_disagree": 0.02077 },2078 {2079 "tau": 29.25,2080 "escalated_frac": 0.9793294270833334,2081 "residual_disagree": 0.02082 },2083 {2084 "tau": 30.25,2085 "escalated_frac": 0.9847005208333334,2086 "residual_disagree": 0.02087 },2088 {2089 "tau": 31.375,2090 "escalated_frac": 0.9895833333333334,2091 "residual_disagree": 0.02092 },2093 {2094 "tau": 32.875,2095 "escalated_frac": 0.9947916666666666,2096 "residual_disagree": 0.02097 },2098 {2099 "tau": 38.375,2100 "escalated_frac": 0.9998372395833334,2101 "residual_disagree": 0.02102 }2103 ],2104 "per_domain": {2105 "code": {2106 "agreement_rate": 0.923828125,2107 "auroc": 0.91594161652301192108 },2109 "math": {2110 "agreement_rate": 0.9462890625,2111 "auroc": 0.93719861150201712112 },2113 "chat": {2114 "agreement_rate": 0.896484375,2115 "auroc": 0.91694927446869732116 },2117 "french": {2118 "agreement_rate": 0.91796875,2119 "auroc": 0.91315856129685912120 },2121 "english": {2122 "agreement_rate": 0.90234375,2123 "auroc": 0.90383658008658012124 },2125 "reasoning": {2126 "agreement_rate": 0.9404296875,2127 "auroc": 0.945661610745112128 }2129 },2130 "escalation_frac_for_99pct": 0.226074218752131 },2132 "8": {2133 "bits": 8,2134 "n_positions": 6144,2135 "agreement_rate": 0.9866536458333334,2136 "mean_kl_ref_q": 0.0017304812790825963,2137 "auroc_margin_predicts_disagreement": 0.980641098888719,2138 "median_margin_agree": 6.5,2139 "median_margin_disagree": 0.125,2140 "escalation_curve": [2141 {2142 "tau": 0.0,2143 "escalated_frac": 0.0,2144 "residual_disagree": 0.0133463541666666662145 },2146 {2147 "tau": 0.0,2148 "escalated_frac": 0.0,2149 "residual_disagree": 0.0133463541666666662150 },2151 {2152 "tau": 0.0,2153 "escalated_frac": 0.0,2154 "residual_disagree": 0.0133463541666666662155 },2156 {2157 "tau": 0.0,2158 "escalated_frac": 0.0,2159 "residual_disagree": 0.0133463541666666662160 },2161 {2162 "tau": 0.125,2163 "escalated_frac": 0.015950520833333332,2164 "residual_disagree": 0.0081380208333333342165 },2166 {2167 "tau": 0.25,2168 "escalated_frac": 0.02099609375,2169 "residual_disagree": 0.0065104166666666672170 },2171 {2172 "tau": 0.25,2173 "escalated_frac": 0.02099609375,2174 "residual_disagree": 0.0065104166666666672175 },2176 {2177 "tau": 0.25,2178 "escalated_frac": 0.02099609375,2179 "residual_disagree": 0.0065104166666666672180 },2181 {2182 "tau": 0.25,2183 "escalated_frac": 0.02099609375,2184 "residual_disagree": 0.0065104166666666672185 },2186 {2187 "tau": 0.25,2188 "escalated_frac": 0.02099609375,2189 "residual_disagree": 0.0065104166666666672190 },2191 {2192 "tau": 0.375,2193 "escalated_frac": 0.047037760416666664,2194 "residual_disagree": 0.001464843752195 },2196 {2197 "tau": 0.5,2198 "escalated_frac": 0.052408854166666664,2199 "residual_disagree": 0.00113932291666666672200 },2201 {2202 "tau": 0.5,2203 "escalated_frac": 0.052408854166666664,2204 "residual_disagree": 0.00113932291666666672205 },2206 {2207 "tau": 0.5,2208 "escalated_frac": 0.052408854166666664,2209 "residual_disagree": 0.00113932291666666672210 },2211 {2212 "tau": 0.5,2213 "escalated_frac": 0.052408854166666664,2214 "residual_disagree": 0.00113932291666666672215 },2216 {2217 "tau": 0.5,2218 "escalated_frac": 0.052408854166666664,2219 "residual_disagree": 0.00113932291666666672220 },2221 {2222 "tau": 0.625,2223 "escalated_frac": 0.07845052083333333,2224 "residual_disagree": 0.00032552083333333332225 },2226 {2227 "tau": 0.75,2228 "escalated_frac": 0.08463541666666667,2229 "residual_disagree": 0.00032552083333333332230 },2231 {2232 "tau": 0.75,2233 "escalated_frac": 0.08463541666666667,2234 "residual_disagree": 0.00032552083333333332235 },2236 {2237 "tau": 0.75,2238 "escalated_frac": 0.08463541666666667,2239 "residual_disagree": 0.00032552083333333332240 },2241 {2242 "tau": 0.75,2243 "escalated_frac": 0.08463541666666667,2244 "residual_disagree": 0.00032552083333333332245 },2246 {2247 "tau": 0.75,2248 "escalated_frac": 0.08463541666666667,2249 "residual_disagree": 0.00032552083333333332250 },2251 {2252 "tau": 0.875,2253 "escalated_frac": 0.10774739583333333,2254 "residual_disagree": 0.02255 },2256 {2257 "tau": 1.0,2258 "escalated_frac": 0.11246744791666667,2259 "residual_disagree": 0.02260 },2261 {2262 "tau": 1.0,2263 "escalated_frac": 0.11246744791666667,2264 "residual_disagree": 0.02265 },2266 {2267 "tau": 1.0,2268 "escalated_frac": 0.11246744791666667,2269 "residual_disagree": 0.02270 },2271 {2272 "tau": 1.0,2273 "escalated_frac": 0.11246744791666667,2274 "residual_disagree": 0.02275 },2276 {2277 "tau": 1.125,2278 "escalated_frac": 0.13509114583333334,2279 "residual_disagree": 0.02280 },2281 {2282 "tau": 1.25,2283 "escalated_frac": 0.140625,2284 "residual_disagree": 0.02285 },2286 {2287 "tau": 1.25,2288 "escalated_frac": 0.140625,2289 "residual_disagree": 0.02290 },2291 {2292 "tau": 1.25,2293 "escalated_frac": 0.140625,2294 "residual_disagree": 0.02295 },2296 {2297 "tau": 1.25,2298 "escalated_frac": 0.140625,2299 "residual_disagree": 0.02300 },2301 {2302 "tau": 1.25,2303 "escalated_frac": 0.140625,2304 "residual_disagree": 0.02305 },2306 {2307 "tau": 1.375,2308 "escalated_frac": 0.16145833333333334,2309 "residual_disagree": 0.02310 },2311 {2312 "tau": 1.5,2313 "escalated_frac": 0.16650390625,2314 "residual_disagree": 0.02315 },2316 {2317 "tau": 1.5,2318 "escalated_frac": 0.16650390625,2319 "residual_disagree": 0.02320 },2321 {2322 "tau": 1.5,2323 "escalated_frac": 0.16650390625,2324 "residual_disagree": 0.02325 },2326 {2327 "tau": 1.5,2328 "escalated_frac": 0.16650390625,2329 "residual_disagree": 0.02330 },2331 {2332 "tau": 1.625,2333 "escalated_frac": 0.1865234375,2334 "residual_disagree": 0.02335 },2336 {2337 "tau": 1.75,2338 "escalated_frac": 0.19156901041666666,2339 "residual_disagree": 0.02340 },2341 {2342 "tau": 1.75,2343 "escalated_frac": 0.19156901041666666,2344 "residual_disagree": 0.02345 },2346 {2347 "tau": 1.75,2348 "escalated_frac": 0.19156901041666666,2349 "residual_disagree": 0.02350 },2351 {2352 "tau": 1.875,2353 "escalated_frac": 0.21012369791666666,2354 "residual_disagree": 0.02355 },2356 {2357 "tau": 2.0,2358 "escalated_frac": 0.21468098958333334,2359 "residual_disagree": 0.02360 },2361 {2362 "tau": 2.0,2363 "escalated_frac": 0.21468098958333334,2364 "residual_disagree": 0.02365 },2366 {2367 "tau": 2.0,2368 "escalated_frac": 0.21468098958333334,2369 "residual_disagree": 0.02370 },2371 {2372 "tau": 2.0,2373 "escalated_frac": 0.21468098958333334,2374 "residual_disagree": 0.02375 },2376 {2377 "tau": 2.125,2378 "escalated_frac": 0.23583984375,2379 "residual_disagree": 0.02380 },2381 {2382 "tau": 2.25,2383 "escalated_frac": 0.23974609375,2384 "residual_disagree": 0.02385 },2386 {2387 "tau": 2.25,2388 "escalated_frac": 0.23974609375,2389 "residual_disagree": 0.02390 },2391 {2392 "tau": 2.25,2393 "escalated_frac": 0.23974609375,2394 "residual_disagree": 0.02395 },2396 {2397 "tau": 2.25,2398 "escalated_frac": 0.23974609375,2399 "residual_disagree": 0.02400 },2401 {2402 "tau": 2.375,2403 "escalated_frac": 0.2579752604166667,2404 "residual_disagree": 0.02405 },2406 {2407 "tau": 2.5,2408 "escalated_frac": 0.2618815104166667,2409 "residual_disagree": 0.02410 },2411 {2412 "tau": 2.5,2413 "escalated_frac": 0.2618815104166667,2414 "residual_disagree": 0.02415 },2416 {2417 "tau": 2.5,2418 "escalated_frac": 0.2618815104166667,2419 "residual_disagree": 0.02420 },2421 {2422 "tau": 2.75,2423 "escalated_frac": 0.28125,2424 "residual_disagree": 0.02425 },2426 {2427 "tau": 2.75,2428 "escalated_frac": 0.28125,2429 "residual_disagree": 0.02430 },2431 {2432 "tau": 2.75,2433 "escalated_frac": 0.28125,2434 "residual_disagree": 0.02435 },2436 {2437 "tau": 2.75,2438 "escalated_frac": 0.28125,2439 "residual_disagree": 0.02440 },2441 {2442 "tau": 2.875,2443 "escalated_frac": 0.2985026041666667,2444 "residual_disagree": 0.02445 },2446 {2447 "tau": 3.0,2448 "escalated_frac": 0.3028971354166667,2449 "residual_disagree": 0.02450 },2451 {2452 "tau": 3.0,2453 "escalated_frac": 0.3028971354166667,2454 "residual_disagree": 0.02455 },2456 {2457 "tau": 3.096105527638173,2458 "escalated_frac": 0.3165690104166667,2459 "residual_disagree": 0.02460 },2461 {2462 "tau": 3.25,2463 "escalated_frac": 0.3196614583333333,2464 "residual_disagree": 0.02465 },2466 {2467 "tau": 3.25,2468 "escalated_frac": 0.3196614583333333,2469 "residual_disagree": 0.02470 },2471 {2472 "tau": 3.25,2473 "escalated_frac": 0.3196614583333333,2474 "residual_disagree": 0.02475 },2476 {2477 "tau": 3.375,2478 "escalated_frac": 0.3361002604166667,2479 "residual_disagree": 0.02480 },2481 {2482 "tau": 3.5,2483 "escalated_frac": 0.3385416666666667,2484 "residual_disagree": 0.02485 },2486 {2487 "tau": 3.5,2488 "escalated_frac": 0.3385416666666667,2489 "residual_disagree": 0.02490 },2491 {2492 "tau": 3.5,2493 "escalated_frac": 0.3385416666666667,2494 "residual_disagree": 0.02495 },2496 {2497 "tau": 3.75,2498 "escalated_frac": 0.3557942708333333,2499 "residual_disagree": 0.02500 },2501 {2502 "tau": 3.75,2503 "escalated_frac": 0.3557942708333333,2504 "residual_disagree": 0.02505 },2506 {2507 "tau": 3.75,2508 "escalated_frac": 0.3557942708333333,2509 "residual_disagree": 0.02510 },2511 {2512 "tau": 4.0,2513 "escalated_frac": 0.37158203125,2514 "residual_disagree": 0.02515 },2516 {2517 "tau": 4.0,2518 "escalated_frac": 0.37158203125,2519 "residual_disagree": 0.02520 },2521 {2522 "tau": 4.0,2523 "escalated_frac": 0.37158203125,2524 "residual_disagree": 0.02525 },2526 {2527 "tau": 4.25,2528 "escalated_frac": 0.38427734375,2529 "residual_disagree": 0.02530 },2531 {2532 "tau": 4.25,2533 "escalated_frac": 0.38427734375,2534 "residual_disagree": 0.02535 },2536 {2537 "tau": 4.5,2538 "escalated_frac": 0.39501953125,2539 "residual_disagree": 0.02540 },2541 {2542 "tau": 4.5,2543 "escalated_frac": 0.39501953125,2544 "residual_disagree": 0.02545 },2546 {2547 "tau": 4.625,2548 "escalated_frac": 0.4052734375,2549 "residual_disagree": 0.02550 },2551 {2552 "tau": 4.75,2553 "escalated_frac": 0.40771484375,2554 "residual_disagree": 0.02555 },2556 {2557 "tau": 4.75,2558 "escalated_frac": 0.40771484375,2559 "residual_disagree": 0.02560 },2561 {2562 "tau": 4.875,2563 "escalated_frac": 0.4207356770833333,2564 "residual_disagree": 0.02565 },2566 {2567 "tau": 5.0,2568 "escalated_frac": 0.423828125,2569 "residual_disagree": 0.02570 },2571 {2572 "tau": 5.0,2573 "escalated_frac": 0.423828125,2574 "residual_disagree": 0.02575 },2576 {2577 "tau": 5.125,2578 "escalated_frac": 0.43603515625,2579 "residual_disagree": 0.02580 },2581 {2582 "tau": 5.25,2583 "escalated_frac": 0.4386393229166667,2584 "residual_disagree": 0.02585 },2586 {2587 "tau": 5.25,2588 "escalated_frac": 0.4386393229166667,2589 "residual_disagree": 0.02590 },2591 {2592 "tau": 5.375,2593 "escalated_frac": 0.44970703125,2594 "residual_disagree": 0.02595 },2596 {2597 "tau": 5.5,2598 "escalated_frac": 0.453125,2599 "residual_disagree": 0.02600 },2601 {2602 "tau": 5.5,2603 "escalated_frac": 0.453125,2604 "residual_disagree": 0.02605 },2606 {2607 "tau": 5.75,2608 "escalated_frac": 0.4666341145833333,2609 "residual_disagree": 0.02610 },2611 {2612 "tau": 5.75,2613 "escalated_frac": 0.4666341145833333,2614 "residual_disagree": 0.02615 },2616 {2617 "tau": 5.875,2618 "escalated_frac": 0.4762369791666667,2619 "residual_disagree": 0.02620 },2621 {2622 "tau": 6.0,2623 "escalated_frac": 0.4786783854166667,2624 "residual_disagree": 0.02625 },2626 {2627 "tau": 6.0,2628 "escalated_frac": 0.4786783854166667,2629 "residual_disagree": 0.02630 },2631 {2632 "tau": 6.25,2633 "escalated_frac": 0.49169921875,2634 "residual_disagree": 0.02635 },2636 {2637 "tau": 6.25,2638 "escalated_frac": 0.49169921875,2639 "residual_disagree": 0.02640 },2641 {2642 "tau": 6.5,2643 "escalated_frac": 0.4990234375,2644 "residual_disagree": 0.02645 },2646 {2647 "tau": 6.5,2648 "escalated_frac": 0.4990234375,2649 "residual_disagree": 0.02650 },2651 {2652 "tau": 6.75,2653 "escalated_frac": 0.5104166666666666,2654 "residual_disagree": 0.02655 },2656 {2657 "tau": 6.75,2658 "escalated_frac": 0.5104166666666666,2659 "residual_disagree": 0.02660 },2661 {2662 "tau": 7.0,2663 "escalated_frac": 0.5193684895833334,2664 "residual_disagree": 0.02665 },2666 {2667 "tau": 7.25,2668 "escalated_frac": 0.5271809895833334,2669 "residual_disagree": 0.02670 },2671 {2672 "tau": 7.25,2673 "escalated_frac": 0.5271809895833334,2674 "residual_disagree": 0.02675 },2676 {2677 "tau": 7.375,2678 "escalated_frac": 0.5359700520833334,2679 "residual_disagree": 0.02680 },2681 {2682 "tau": 7.5,2683 "escalated_frac": 0.5380859375,2684 "residual_disagree": 0.02685 },2686 {2687 "tau": 7.75,2688 "escalated_frac": 0.5462239583333334,2689 "residual_disagree": 0.02690 },2691 {2692 "tau": 7.953517587939757,2693 "escalated_frac": 0.552734375,2694 "residual_disagree": 0.02695 },2696 {2697 "tau": 8.0,2698 "escalated_frac": 0.552734375,2699 "residual_disagree": 0.02700 },2701 {2702 "tau": 8.25,2703 "escalated_frac": 0.5616861979166666,2704 "residual_disagree": 0.02705 },2706 {2707 "tau": 8.25,2708 "escalated_frac": 0.5616861979166666,2709 "residual_disagree": 0.02710 },2711 {2712 "tau": 8.5,2713 "escalated_frac": 0.56982421875,2714 "residual_disagree": 0.02715 },2716 {2717 "tau": 8.625,2718 "escalated_frac": 0.576171875,2719 "residual_disagree": 0.02720 },2721 {2722 "tau": 8.75,2723 "escalated_frac": 0.5779622395833334,2724 "residual_disagree": 0.02725 },2726 {2727 "tau": 9.0,2728 "escalated_frac": 0.5847981770833334,2729 "residual_disagree": 0.02730 },2731 {2732 "tau": 9.25,2733 "escalated_frac": 0.5914713541666666,2734 "residual_disagree": 0.02735 },2736 {2737 "tau": 9.5,2738 "escalated_frac": 0.5978190104166666,2739 "residual_disagree": 0.02740 },2741 {2742 "tau": 9.5,2743 "escalated_frac": 0.5978190104166666,2744 "residual_disagree": 0.02745 },2746 {2747 "tau": 9.75,2748 "escalated_frac": 0.6046549479166666,2749 "residual_disagree": 0.02750 },2751 {2752 "tau": 10.0,2753 "escalated_frac": 0.61279296875,2754 "residual_disagree": 0.02755 },2756 {2757 "tau": 10.24120603015075,2758 "escalated_frac": 0.6180013020833334,2759 "residual_disagree": 0.02760 },2761 {2762 "tau": 10.25,2763 "escalated_frac": 0.6180013020833334,2764 "residual_disagree": 0.02765 },2766 {2767 "tau": 10.5,2768 "escalated_frac": 0.625,2769 "residual_disagree": 0.02770 },2771 {2772 "tau": 10.75,2773 "escalated_frac": 0.6321614583333334,2774 "residual_disagree": 0.02775 },2776 {2777 "tau": 10.875,2778 "escalated_frac": 0.6370442708333334,2779 "residual_disagree": 0.02780 },2781 {2782 "tau": 11.0,2783 "escalated_frac": 0.638671875,2784 "residual_disagree": 0.02785 },2786 {2787 "tau": 11.25,2788 "escalated_frac": 0.6476236979166666,2789 "residual_disagree": 0.02790 },2791 {2792 "tau": 11.5,2793 "escalated_frac": 0.6529947916666666,2794 "residual_disagree": 0.02795 },2796 {2797 "tau": 11.610552763819044,2798 "escalated_frac": 0.658203125,2799 "residual_disagree": 0.02800 },2801 {2802 "tau": 11.75,2803 "escalated_frac": 0.65966796875,2804 "residual_disagree": 0.02805 },2806 {2807 "tau": 12.0,2808 "escalated_frac": 0.6681315104166666,2809 "residual_disagree": 0.02810 },2811 {2812 "tau": 12.25,2813 "escalated_frac": 0.6730143229166666,2814 "residual_disagree": 0.02815 },2816 {2817 "tau": 12.375,2818 "escalated_frac": 0.6774088541666666,2819 "residual_disagree": 0.02820 },2821 {2822 "tau": 12.5,2823 "escalated_frac": 0.6796875,2824 "residual_disagree": 0.02825 },2826 {2827 "tau": 12.75,2828 "escalated_frac": 0.6875,2829 "residual_disagree": 0.02830 },2831 {2832 "tau": 12.875,2833 "escalated_frac": 0.6917317708333334,2834 "residual_disagree": 0.02835 },2836 {2837 "tau": 13.0,2838 "escalated_frac": 0.6941731770833334,2839 "residual_disagree": 0.02840 },2841 {2842 "tau": 13.25,2843 "escalated_frac": 0.7014973958333334,2844 "residual_disagree": 0.02845 },2846 {2847 "tau": 13.375,2848 "escalated_frac": 0.7073567708333334,2849 "residual_disagree": 0.02850 },2851 {2852 "tau": 13.5,2853 "escalated_frac": 0.7097981770833334,2854 "residual_disagree": 0.02855 },2856 {2857 "tau": 13.75,2858 "escalated_frac": 0.7169596354166666,2859 "residual_disagree": 0.02860 },2861 {2862 "tau": 14.0,2863 "escalated_frac": 0.7228190104166666,2864 "residual_disagree": 0.02865 },2866 {2867 "tau": 14.125,2868 "escalated_frac": 0.72705078125,2869 "residual_disagree": 0.02870 },2871 {2872 "tau": 14.375,2873 "escalated_frac": 0.7333984375,2874 "residual_disagree": 0.02875 },2876 {2877 "tau": 14.5,2878 "escalated_frac": 0.7353515625,2879 "residual_disagree": 0.02880 },2881 {2882 "tau": 14.75,2883 "escalated_frac": 0.7433268229166666,2884 "residual_disagree": 0.02885 },2886 {2887 "tau": 14.941582914572905,2888 "escalated_frac": 0.7486979166666666,2889 "residual_disagree": 0.02890 },2891 {2892 "tau": 15.125,2893 "escalated_frac": 0.7522786458333334,2894 "residual_disagree": 0.02895 },2896 {2897 "tau": 15.375,2898 "escalated_frac": 0.7584635416666666,2899 "residual_disagree": 0.02900 },2901 {2902 "tau": 15.625,2903 "escalated_frac": 0.7628580729166666,2904 "residual_disagree": 0.02905 },2906 {2907 "tau": 15.875,2908 "escalated_frac": 0.7683919270833334,2909 "residual_disagree": 0.02910 },2911 {2912 "tau": 16.0,2913 "escalated_frac": 0.7706705729166666,2914 "residual_disagree": 0.02915 },2916 {2917 "tau": 16.25,2918 "escalated_frac": 0.7775065104166666,2919 "residual_disagree": 0.02920 },2921 {2922 "tau": 16.5,2923 "escalated_frac": 0.7828776041666666,2924 "residual_disagree": 0.02925 },2926 {2927 "tau": 16.75,2928 "escalated_frac": 0.7884114583333334,2929 "residual_disagree": 0.02930 },2931 {2932 "tau": 17.0,2933 "escalated_frac": 0.7937825520833334,2934 "residual_disagree": 0.02935 },2936 {2937 "tau": 17.125,2938 "escalated_frac": 0.796875,2939 "residual_disagree": 0.02940 },2941 {2942 "tau": 17.25,2943 "escalated_frac": 0.7998046875,2944 "residual_disagree": 0.02945 },2946 {2947 "tau": 17.625,2948 "escalated_frac": 0.8084309895833334,2949 "residual_disagree": 0.02950 },2951 {2952 "tau": 17.875,2953 "escalated_frac": 0.8116861979166666,2954 "residual_disagree": 0.02955 },2956 {2957 "tau": 18.125,2958 "escalated_frac": 0.8185221354166666,2959 "residual_disagree": 0.02960 },2961 {2962 "tau": 18.321608040201,2963 "escalated_frac": 0.8240559895833334,2964 "residual_disagree": 0.02965 },2966 {2967 "tau": 18.625,2968 "escalated_frac": 0.8287760416666666,2969 "residual_disagree": 0.02970 },2971 {2972 "tau": 18.75,2973 "escalated_frac": 0.8304036458333334,2974 "residual_disagree": 0.02975 },2976 {2977 "tau": 19.125,2978 "escalated_frac": 0.8390299479166666,2979 "residual_disagree": 0.02980 },2981 {2982 "tau": 19.25,2983 "escalated_frac": 0.8416341145833334,2984 "residual_disagree": 0.02985 },2986 {2987 "tau": 19.5,2988 "escalated_frac": 0.8483072916666666,2989 "residual_disagree": 0.02990 },2991 {2992 "tau": 19.75,2993 "escalated_frac": 0.8527018229166666,2994 "residual_disagree": 0.02995 },2996 {2997 "tau": 20.0,2998 "escalated_frac": 0.85791015625,2999 "residual_disagree": 0.03000 },3001 {3002 "tau": 20.25,3003 "escalated_frac": 0.8636067708333334,3004 "residual_disagree": 0.03005 },3006 {3007 "tau": 20.5,3008 "escalated_frac": 0.8673502604166666,3009 "residual_disagree": 0.03010 },3011 {3012 "tau": 20.875,3013 "escalated_frac": 0.8723958333333334,3014 "residual_disagree": 0.03015 },3016 {3017 "tau": 21.25,3018 "escalated_frac": 0.87890625,3019 "residual_disagree": 0.03020 },3021 {3022 "tau": 21.5,3023 "escalated_frac": 0.8837890625,3024 "residual_disagree": 0.03025 },3026 {3027 "tau": 21.75,3028 "escalated_frac": 0.8880208333333334,3029 "residual_disagree": 0.03030 },3031 {3032 "tau": 22.0,3033 "escalated_frac": 0.8929036458333334,3034 "residual_disagree": 0.03035 },3036 {3037 "tau": 22.25,3038 "escalated_frac": 0.8976236979166666,3039 "residual_disagree": 0.03040 },3041 {3042 "tau": 22.625,3043 "escalated_frac": 0.904296875,3044 "residual_disagree": 0.03045 },3046 {3047 "tau": 22.875,3048 "escalated_frac": 0.908203125,3049 "residual_disagree": 0.03050 },3051 {3052 "tau": 23.125,3053 "escalated_frac": 0.9134114583333334,3054 "residual_disagree": 0.03055 },3056 {3057 "tau": 23.5,3058 "escalated_frac": 0.9178059895833334,3059 "residual_disagree": 0.03060 },3061 {3062 "tau": 23.75,3063 "escalated_frac": 0.9236653645833334,3064 "residual_disagree": 0.03065 },3066 {3067 "tau": 24.0,3068 "escalated_frac": 0.9280598958333334,3069 "residual_disagree": 0.03070 },3071 {3072 "tau": 24.25,3073 "escalated_frac": 0.9322916666666666,3074 "residual_disagree": 0.03075 },3076 {3077 "tau": 24.75,3078 "escalated_frac": 0.939453125,3079 "residual_disagree": 0.03080 },3081 {3082 "tau": 25.0,3083 "escalated_frac": 0.9436848958333334,3084 "residual_disagree": 0.03085 },3086 {3087 "tau": 25.5,3088 "escalated_frac": 0.9493815104166666,3089 "residual_disagree": 0.03090 },3091 {3092 "tau": 25.771984924623098,3093 "escalated_frac": 0.9547526041666666,3094 "residual_disagree": 0.03095 },3096 {3097 "tau": 26.375,3098 "escalated_frac": 0.9596354166666666,3099 "residual_disagree": 0.03100 },3101 {3102 "tau": 27.125,3103 "escalated_frac": 0.96435546875,3104 "residual_disagree": 0.03105 },3106 {3107 "tau": 27.75,3108 "escalated_frac": 0.9694010416666666,3109 "residual_disagree": 0.03110 },3111 {3112 "tau": 28.375,3113 "escalated_frac": 0.9739583333333334,3114 "residual_disagree": 0.03115 },3116 {3117 "tau": 29.315326633165796,3118 "escalated_frac": 0.9798177083333334,3119 "residual_disagree": 0.03120 },3121 {3122 "tau": 30.375,3123 "escalated_frac": 0.9845377604166666,3124 "residual_disagree": 0.03125 },3126 {3127 "tau": 31.375,3128 "escalated_frac": 0.9892578125,3129 "residual_disagree": 0.03130 },3131 {3132 "tau": 33.14133165829139,3133 "escalated_frac": 0.9949544270833334,3134 "residual_disagree": 0.03135 },3136 {3137 "tau": 39.0625,3138 "escalated_frac": 0.9998372395833334,3139 "residual_disagree": 0.03140 }3141 ],3142 "per_domain": {3143 "code": {3144 "agreement_rate": 0.98828125,3145 "auroc": 0.98353096179183133146 },3147 "math": {3148 "agreement_rate": 0.9921875,3149 "auroc": 0.99311023622047253150 },3151 "chat": {3152 "agreement_rate": 0.9765625,3153 "auroc": 0.9676253154 },3155 "french": {3156 "agreement_rate": 0.98828125,3157 "auroc": 0.97319664031620563158 },3159 "english": {3160 "agreement_rate": 0.982421875,3161 "auroc": 0.98426110006626913162 },3163 "reasoning": {3164 "agreement_rate": 0.9921875,3165 "auroc": 0.98339074803149613166 }3167 },3168 "escalation_frac_for_99pct": 0.0159505208333333323169 }3170 }3171}