spb/localvm-research Public License
Running LLMs larger than memory on a consumer Mac — falsification-driven research: margin-gated deferred refinement, out-of-core verification on Apple Silicon. TR-01 published.
Python 63.2%
JavaScript 23.5%
CSS 11.8%
Shell 0.9%
Makefile 0.5%
1{2 "experiment": "expG_decision_stability",3 "author": "Simon-Pierre Boucher",4 "contact": "contact@spboucher.ai",5 "manifest": {6 "author": "Simon-Pierre Boucher",7 "contact": "contact@spboucher.ai",8 "project": "localvm-research",9 "collected_utc": "2026-08-12T04:27:44.503861+00:00",10 "chip": {11 "brand": "Apple M5 Max",12 "arch": "arm64",13 "cores_total": 18,14 "cores_performance": 6,15 "cores_efficiency": 12,16 "gpu_cores": 4017 },18 "memory": {19 "unified_bytes": 51539607552,20 "unified_gb": 48.0,21 "pagesize": 1638422 },23 "ssd": {24 "model": "APPLE SSD AP2048Z",25 "size": "2 TB",26 "smart_status": "Verified"27 },28 "os": {29 "product": "macOS",30 "version": "27.0",31 "build": "26A5388g",32 "kernel": "27.0.0"33 },34 "software": {35 "python": "3.14.4",36 "mlx": "0.32.0",37 "mlx_lm": "0.31.3",38 "torch": null,39 "numpy": "2.5.2"40 },41 "git": {42 "commit": "2b7d23b36999ff09c3e6ac03a730575e71e79650",43 "dirty_tree": true44 },45 "thermal_level_at_collect": null46 },47 "config": {48 "model": "mlx-community/Qwen3-1.7B-bf16",49 "gen_tokens": 128,50 "bits": "2,3,4,8",51 "per_domain": 8,52 "group_size": 6453 },54 "n_trajectories": 48,55 "results_by_bits": {56 "2": {57 "bits": 2,58 "n_positions": 6144,59 "agreement_rate": 0.029622395833333332,60 "mean_kl_ref_q": 11.892386436462402,61 "auroc_margin_predicts_disagreement": 0.6048301329666643,62 "median_margin_agree": 1.5,63 "median_margin_disagree": 1.0,64 "escalation_curve": [65 {66 "tau": 0.0,67 "escalated_frac": 0.0,68 "residual_disagree": 0.970377604166666669 },70 {71 "tau": 0.0,72 "escalated_frac": 0.0,73 "residual_disagree": 0.970377604166666674 },75 {76 "tau": 0.0,77 "escalated_frac": 0.0,78 "residual_disagree": 0.970377604166666679 },80 {81 "tau": 0.0,82 "escalated_frac": 0.0,83 "residual_disagree": 0.970377604166666684 },85 {86 "tau": 0.0,87 "escalated_frac": 0.0,88 "residual_disagree": 0.970377604166666689 },90 {91 "tau": 0.0,92 "escalated_frac": 0.0,93 "residual_disagree": 0.970377604166666694 },95 {96 "tau": 0.0,97 "escalated_frac": 0.0,98 "residual_disagree": 0.970377604166666699 },100 {101 "tau": 0.0,102 "escalated_frac": 0.0,103 "residual_disagree": 0.9703776041666666104 },105 {106 "tau": 0.0625,107 "escalated_frac": 0.038411458333333336,108 "residual_disagree": 0.9327799479166666109 },110 {111 "tau": 0.125,112 "escalated_frac": 0.043619791666666664,113 "residual_disagree": 0.927734375114 },115 {116 "tau": 0.125,117 "escalated_frac": 0.043619791666666664,118 "residual_disagree": 0.927734375119 },120 {121 "tau": 0.125,122 "escalated_frac": 0.043619791666666664,123 "residual_disagree": 0.927734375124 },125 {126 "tau": 0.125,127 "escalated_frac": 0.043619791666666664,128 "residual_disagree": 0.927734375129 },130 {131 "tau": 0.125,132 "escalated_frac": 0.043619791666666664,133 "residual_disagree": 0.927734375134 },135 {136 "tau": 0.125,137 "escalated_frac": 0.043619791666666664,138 "residual_disagree": 0.927734375139 },140 {141 "tau": 0.125,142 "escalated_frac": 0.043619791666666664,143 "residual_disagree": 0.927734375144 },145 {146 "tau": 0.125,147 "escalated_frac": 0.043619791666666664,148 "residual_disagree": 0.927734375149 },150 {151 "tau": 0.125,152 "escalated_frac": 0.043619791666666664,153 "residual_disagree": 0.927734375154 },155 {156 "tau": 0.125,157 "escalated_frac": 0.043619791666666664,158 "residual_disagree": 0.927734375159 },160 {161 "tau": 0.125,162 "escalated_frac": 0.043619791666666664,163 "residual_disagree": 0.927734375164 },165 {166 "tau": 0.125,167 "escalated_frac": 0.043619791666666664,168 "residual_disagree": 0.927734375169 },170 {171 "tau": 0.125,172 "escalated_frac": 0.043619791666666664,173 "residual_disagree": 0.927734375174 },175 {176 "tau": 0.125,177 "escalated_frac": 0.043619791666666664,178 "residual_disagree": 0.927734375179 },180 {181 "tau": 0.1875,182 "escalated_frac": 0.11328125,183 "residual_disagree": 0.8601888020833334184 },185 {186 "tau": 0.25,187 "escalated_frac": 0.119140625,188 "residual_disagree": 0.8543294270833334189 },190 {191 "tau": 0.25,192 "escalated_frac": 0.119140625,193 "residual_disagree": 0.8543294270833334194 },195 {196 "tau": 0.25,197 "escalated_frac": 0.119140625,198 "residual_disagree": 0.8543294270833334199 },200 {201 "tau": 0.25,202 "escalated_frac": 0.119140625,203 "residual_disagree": 0.8543294270833334204 },205 {206 "tau": 0.25,207 "escalated_frac": 0.119140625,208 "residual_disagree": 0.8543294270833334209 },210 {211 "tau": 0.25,212 "escalated_frac": 0.119140625,213 "residual_disagree": 0.8543294270833334214 },215 {216 "tau": 0.25,217 "escalated_frac": 0.119140625,218 "residual_disagree": 0.8543294270833334219 },220 {221 "tau": 0.25,222 "escalated_frac": 0.119140625,223 "residual_disagree": 0.8543294270833334224 },225 {226 "tau": 0.25,227 "escalated_frac": 0.119140625,228 "residual_disagree": 0.8543294270833334229 },230 {231 "tau": 0.25,232 "escalated_frac": 0.119140625,233 "residual_disagree": 0.8543294270833334234 },235 {236 "tau": 0.25,237 "escalated_frac": 0.119140625,238 "residual_disagree": 0.8543294270833334239 },240 {241 "tau": 0.25,242 "escalated_frac": 0.119140625,243 "residual_disagree": 0.8543294270833334244 },245 {246 "tau": 0.25,247 "escalated_frac": 0.119140625,248 "residual_disagree": 0.8543294270833334249 },250 {251 "tau": 0.25,252 "escalated_frac": 0.119140625,253 "residual_disagree": 0.8543294270833334254 },255 {256 "tau": 0.3125,257 "escalated_frac": 0.18912760416666666,258 "residual_disagree": 0.7853190104166666259 },260 {261 "tau": 0.375,262 "escalated_frac": 0.19352213541666666,263 "residual_disagree": 0.7809244791666666264 },265 {266 "tau": 0.375,267 "escalated_frac": 0.19352213541666666,268 "residual_disagree": 0.7809244791666666269 },270 {271 "tau": 0.375,272 "escalated_frac": 0.19352213541666666,273 "residual_disagree": 0.7809244791666666274 },275 {276 "tau": 0.375,277 "escalated_frac": 0.19352213541666666,278 "residual_disagree": 0.7809244791666666279 },280 {281 "tau": 0.375,282 "escalated_frac": 0.19352213541666666,283 "residual_disagree": 0.7809244791666666284 },285 {286 "tau": 0.375,287 "escalated_frac": 0.19352213541666666,288 "residual_disagree": 0.7809244791666666289 },290 {291 "tau": 0.375,292 "escalated_frac": 0.19352213541666666,293 "residual_disagree": 0.7809244791666666294 },295 {296 "tau": 0.375,297 "escalated_frac": 0.19352213541666666,298 "residual_disagree": 0.7809244791666666299 },300 {301 "tau": 0.375,302 "escalated_frac": 0.19352213541666666,303 "residual_disagree": 0.7809244791666666304 },305 {306 "tau": 0.375,307 "escalated_frac": 0.19352213541666666,308 "residual_disagree": 0.7809244791666666309 },310 {311 "tau": 0.375,312 "escalated_frac": 0.19352213541666666,313 "residual_disagree": 0.7809244791666666314 },315 {316 "tau": 0.375,317 "escalated_frac": 0.19352213541666666,318 "residual_disagree": 0.7809244791666666319 },320 {321 "tau": 0.375,322 "escalated_frac": 0.19352213541666666,323 "residual_disagree": 0.7809244791666666324 },325 {326 "tau": 0.4375,327 "escalated_frac": 0.25830078125,328 "residual_disagree": 0.7176106770833334329 },330 {331 "tau": 0.5,332 "escalated_frac": 0.2630208333333333,333 "residual_disagree": 0.712890625334 },335 {336 "tau": 0.5,337 "escalated_frac": 0.2630208333333333,338 "residual_disagree": 0.712890625339 },340 {341 "tau": 0.5,342 "escalated_frac": 0.2630208333333333,343 "residual_disagree": 0.712890625344 },345 {346 "tau": 0.5,347 "escalated_frac": 0.2630208333333333,348 "residual_disagree": 0.712890625349 },350 {351 "tau": 0.5,352 "escalated_frac": 0.2630208333333333,353 "residual_disagree": 0.712890625354 },355 {356 "tau": 0.5,357 "escalated_frac": 0.2630208333333333,358 "residual_disagree": 0.712890625359 },360 {361 "tau": 0.5,362 "escalated_frac": 0.2630208333333333,363 "residual_disagree": 0.712890625364 },365 {366 "tau": 0.5,367 "escalated_frac": 0.2630208333333333,368 "residual_disagree": 0.712890625369 },370 {371 "tau": 0.5,372 "escalated_frac": 0.2630208333333333,373 "residual_disagree": 0.712890625374 },375 {376 "tau": 0.5,377 "escalated_frac": 0.2630208333333333,378 "residual_disagree": 0.712890625379 },380 {381 "tau": 0.5,382 "escalated_frac": 0.2630208333333333,383 "residual_disagree": 0.712890625384 },385 {386 "tau": 0.5,387 "escalated_frac": 0.2630208333333333,388 "residual_disagree": 0.712890625389 },390 {391 "tau": 0.5625,392 "escalated_frac": 0.3224283854166667,393 "residual_disagree": 0.65478515625394 },395 {396 "tau": 0.625,397 "escalated_frac": 0.328125,398 "residual_disagree": 0.6490885416666666399 },400 {401 "tau": 0.625,402 "escalated_frac": 0.328125,403 "residual_disagree": 0.6490885416666666404 },405 {406 "tau": 0.625,407 "escalated_frac": 0.328125,408 "residual_disagree": 0.6490885416666666409 },410 {411 "tau": 0.625,412 "escalated_frac": 0.328125,413 "residual_disagree": 0.6490885416666666414 },415 {416 "tau": 0.625,417 "escalated_frac": 0.328125,418 "residual_disagree": 0.6490885416666666419 },420 {421 "tau": 0.625,422 "escalated_frac": 0.328125,423 "residual_disagree": 0.6490885416666666424 },425 {426 "tau": 0.625,427 "escalated_frac": 0.328125,428 "residual_disagree": 0.6490885416666666429 },430 {431 "tau": 0.625,432 "escalated_frac": 0.328125,433 "residual_disagree": 0.6490885416666666434 },435 {436 "tau": 0.625,437 "escalated_frac": 0.328125,438 "residual_disagree": 0.6490885416666666439 },440 {441 "tau": 0.625,442 "escalated_frac": 0.328125,443 "residual_disagree": 0.6490885416666666444 },445 {446 "tau": 0.625,447 "escalated_frac": 0.328125,448 "residual_disagree": 0.6490885416666666449 },450 {451 "tau": 0.75,452 "escalated_frac": 0.3841145833333333,453 "residual_disagree": 0.5944010416666666454 },455 {456 "tau": 0.75,457 "escalated_frac": 0.3841145833333333,458 "residual_disagree": 0.5944010416666666459 },460 {461 "tau": 0.75,462 "escalated_frac": 0.3841145833333333,463 "residual_disagree": 0.5944010416666666464 },465 {466 "tau": 0.75,467 "escalated_frac": 0.3841145833333333,468 "residual_disagree": 0.5944010416666666469 },470 {471 "tau": 0.75,472 "escalated_frac": 0.3841145833333333,473 "residual_disagree": 0.5944010416666666474 },475 {476 "tau": 0.75,477 "escalated_frac": 0.3841145833333333,478 "residual_disagree": 0.5944010416666666479 },480 {481 "tau": 0.75,482 "escalated_frac": 0.3841145833333333,483 "residual_disagree": 0.5944010416666666484 },485 {486 "tau": 0.75,487 "escalated_frac": 0.3841145833333333,488 "residual_disagree": 0.5944010416666666489 },490 {491 "tau": 0.75,492 "escalated_frac": 0.3841145833333333,493 "residual_disagree": 0.5944010416666666494 },495 {496 "tau": 0.75,497 "escalated_frac": 0.3841145833333333,498 "residual_disagree": 0.5944010416666666499 },500 {501 "tau": 0.8125,502 "escalated_frac": 0.4353841145833333,503 "residual_disagree": 0.5445963541666666504 },505 {506 "tau": 0.875,507 "escalated_frac": 0.4383138020833333,508 "residual_disagree": 0.5416666666666666509 },510 {511 "tau": 0.875,512 "escalated_frac": 0.4383138020833333,513 "residual_disagree": 0.5416666666666666514 },515 {516 "tau": 0.875,517 "escalated_frac": 0.4383138020833333,518 "residual_disagree": 0.5416666666666666519 },520 {521 "tau": 0.875,522 "escalated_frac": 0.4383138020833333,523 "residual_disagree": 0.5416666666666666524 },525 {526 "tau": 0.875,527 "escalated_frac": 0.4383138020833333,528 "residual_disagree": 0.5416666666666666529 },530 {531 "tau": 0.875,532 "escalated_frac": 0.4383138020833333,533 "residual_disagree": 0.5416666666666666534 },535 {536 "tau": 0.875,537 "escalated_frac": 0.4383138020833333,538 "residual_disagree": 0.5416666666666666539 },540 {541 "tau": 0.875,542 "escalated_frac": 0.4383138020833333,543 "residual_disagree": 0.5416666666666666544 },545 {546 "tau": 0.875,547 "escalated_frac": 0.4383138020833333,548 "residual_disagree": 0.5416666666666666549 },550 {551 "tau": 0.9579145728643255,552 "escalated_frac": 0.4874674479166667,553 "residual_disagree": 0.4931640625554 },555 {556 "tau": 1.0,557 "escalated_frac": 0.4874674479166667,558 "residual_disagree": 0.4931640625559 },560 {561 "tau": 1.0,562 "escalated_frac": 0.4874674479166667,563 "residual_disagree": 0.4931640625564 },565 {566 "tau": 1.0,567 "escalated_frac": 0.4874674479166667,568 "residual_disagree": 0.4931640625569 },570 {571 "tau": 1.0,572 "escalated_frac": 0.4874674479166667,573 "residual_disagree": 0.4931640625574 },575 {576 "tau": 1.0,577 "escalated_frac": 0.4874674479166667,578 "residual_disagree": 0.4931640625579 },580 {581 "tau": 1.0,582 "escalated_frac": 0.4874674479166667,583 "residual_disagree": 0.4931640625584 },585 {586 "tau": 1.0,587 "escalated_frac": 0.4874674479166667,588 "residual_disagree": 0.4931640625589 },590 {591 "tau": 1.0,592 "escalated_frac": 0.4874674479166667,593 "residual_disagree": 0.4931640625594 },595 {596 "tau": 1.125,597 "escalated_frac": 0.5319010416666666,598 "residual_disagree": 0.4495442708333333599 },600 {601 "tau": 1.125,602 "escalated_frac": 0.5319010416666666,603 "residual_disagree": 0.4495442708333333604 },605 {606 "tau": 1.125,607 "escalated_frac": 0.5319010416666666,608 "residual_disagree": 0.4495442708333333609 },610 {611 "tau": 1.125,612 "escalated_frac": 0.5319010416666666,613 "residual_disagree": 0.4495442708333333614 },615 {616 "tau": 1.125,617 "escalated_frac": 0.5319010416666666,618 "residual_disagree": 0.4495442708333333619 },620 {621 "tau": 1.125,622 "escalated_frac": 0.5319010416666666,623 "residual_disagree": 0.4495442708333333624 },625 {626 "tau": 1.125,627 "escalated_frac": 0.5319010416666666,628 "residual_disagree": 0.4495442708333333629 },630 {631 "tau": 1.1875,632 "escalated_frac": 0.56689453125,633 "residual_disagree": 0.41552734375634 },635 {636 "tau": 1.25,637 "escalated_frac": 0.5686848958333334,638 "residual_disagree": 0.4137369791666667639 },640 {641 "tau": 1.25,642 "escalated_frac": 0.5686848958333334,643 "residual_disagree": 0.4137369791666667644 },645 {646 "tau": 1.25,647 "escalated_frac": 0.5686848958333334,648 "residual_disagree": 0.4137369791666667649 },650 {651 "tau": 1.25,652 "escalated_frac": 0.5686848958333334,653 "residual_disagree": 0.4137369791666667654 },655 {656 "tau": 1.25,657 "escalated_frac": 0.5686848958333334,658 "residual_disagree": 0.4137369791666667659 },660 {661 "tau": 1.25,662 "escalated_frac": 0.5686848958333334,663 "residual_disagree": 0.4137369791666667664 },665 {666 "tau": 1.375,667 "escalated_frac": 0.6022135416666666,668 "residual_disagree": 0.3810221354166667669 },670 {671 "tau": 1.375,672 "escalated_frac": 0.6022135416666666,673 "residual_disagree": 0.3810221354166667674 },675 {676 "tau": 1.375,677 "escalated_frac": 0.6022135416666666,678 "residual_disagree": 0.3810221354166667679 },680 {681 "tau": 1.375,682 "escalated_frac": 0.6022135416666666,683 "residual_disagree": 0.3810221354166667684 },685 {686 "tau": 1.375,687 "escalated_frac": 0.6022135416666666,688 "residual_disagree": 0.3810221354166667689 },690 {691 "tau": 1.375,692 "escalated_frac": 0.6022135416666666,693 "residual_disagree": 0.3810221354166667694 },695 {696 "tau": 1.4086055276381728,697 "escalated_frac": 0.6331380208333334,698 "residual_disagree": 0.3518880208333333699 },700 {701 "tau": 1.5,702 "escalated_frac": 0.6350911458333334,703 "residual_disagree": 0.3499348958333333704 },705 {706 "tau": 1.5,707 "escalated_frac": 0.6350911458333334,708 "residual_disagree": 0.3499348958333333709 },710 {711 "tau": 1.5,712 "escalated_frac": 0.6350911458333334,713 "residual_disagree": 0.3499348958333333714 },715 {716 "tau": 1.5,717 "escalated_frac": 0.6350911458333334,718 "residual_disagree": 0.3499348958333333719 },720 {721 "tau": 1.5,722 "escalated_frac": 0.6350911458333334,723 "residual_disagree": 0.3499348958333333724 },725 {726 "tau": 1.5625,727 "escalated_frac": 0.6629231770833334,728 "residual_disagree": 0.3229166666666667729 },730 {731 "tau": 1.625,732 "escalated_frac": 0.6640625,733 "residual_disagree": 0.32177734375734 },735 {736 "tau": 1.625,737 "escalated_frac": 0.6640625,738 "residual_disagree": 0.32177734375739 },740 {741 "tau": 1.625,742 "escalated_frac": 0.6640625,743 "residual_disagree": 0.32177734375744 },745 {746 "tau": 1.625,747 "escalated_frac": 0.6640625,748 "residual_disagree": 0.32177734375749 },750 {751 "tau": 1.625,752 "escalated_frac": 0.6640625,753 "residual_disagree": 0.32177734375754 },755 {756 "tau": 1.625,757 "escalated_frac": 0.6640625,758 "residual_disagree": 0.32177734375759 },760 {761 "tau": 1.75,762 "escalated_frac": 0.697265625,763 "residual_disagree": 0.2898763020833333764 },765 {766 "tau": 1.75,767 "escalated_frac": 0.697265625,768 "residual_disagree": 0.2898763020833333769 },770 {771 "tau": 1.75,772 "escalated_frac": 0.697265625,773 "residual_disagree": 0.2898763020833333774 },775 {776 "tau": 1.75,777 "escalated_frac": 0.697265625,778 "residual_disagree": 0.2898763020833333779 },780 {781 "tau": 1.75,782 "escalated_frac": 0.697265625,783 "residual_disagree": 0.2898763020833333784 },785 {786 "tau": 1.875,787 "escalated_frac": 0.7234700520833334,788 "residual_disagree": 0.2646484375789 },790 {791 "tau": 1.875,792 "escalated_frac": 0.7234700520833334,793 "residual_disagree": 0.2646484375794 },795 {796 "tau": 1.875,797 "escalated_frac": 0.7234700520833334,798 "residual_disagree": 0.2646484375799 },800 {801 "tau": 1.875,802 "escalated_frac": 0.7234700520833334,803 "residual_disagree": 0.2646484375804 },805 {806 "tau": 1.875,807 "escalated_frac": 0.7234700520833334,808 "residual_disagree": 0.2646484375809 },810 {811 "tau": 2.0,812 "escalated_frac": 0.7451171875,813 "residual_disagree": 0.24348958333333334814 },815 {816 "tau": 2.0,817 "escalated_frac": 0.7451171875,818 "residual_disagree": 0.24348958333333334819 },820 {821 "tau": 2.0,822 "escalated_frac": 0.7451171875,823 "residual_disagree": 0.24348958333333334824 },825 {826 "tau": 2.0,827 "escalated_frac": 0.7451171875,828 "residual_disagree": 0.24348958333333334829 },830 {831 "tau": 2.125,832 "escalated_frac": 0.76708984375,833 "residual_disagree": 0.22184244791666666834 },835 {836 "tau": 2.125,837 "escalated_frac": 0.76708984375,838 "residual_disagree": 0.22184244791666666839 },840 {841 "tau": 2.125,842 "escalated_frac": 0.76708984375,843 "residual_disagree": 0.22184244791666666844 },845 {846 "tau": 2.125,847 "escalated_frac": 0.76708984375,848 "residual_disagree": 0.22184244791666666849 },850 {851 "tau": 2.25,852 "escalated_frac": 0.787109375,853 "residual_disagree": 0.20263671875854 },855 {856 "tau": 2.25,857 "escalated_frac": 0.787109375,858 "residual_disagree": 0.20263671875859 },860 {861 "tau": 2.25,862 "escalated_frac": 0.787109375,863 "residual_disagree": 0.20263671875864 },865 {866 "tau": 2.375,867 "escalated_frac": 0.8030598958333334,868 "residual_disagree": 0.18701171875869 },870 {871 "tau": 2.375,872 "escalated_frac": 0.8030598958333334,873 "residual_disagree": 0.18701171875874 },875 {876 "tau": 2.375,877 "escalated_frac": 0.8030598958333334,878 "residual_disagree": 0.18701171875879 },880 {881 "tau": 2.375,882 "escalated_frac": 0.8030598958333334,883 "residual_disagree": 0.18701171875884 },885 {886 "tau": 2.5,887 "escalated_frac": 0.8204752604166666,888 "residual_disagree": 0.169921875889 },890 {891 "tau": 2.5,892 "escalated_frac": 0.8204752604166666,893 "residual_disagree": 0.169921875894 },895 {896 "tau": 2.5,897 "escalated_frac": 0.8204752604166666,898 "residual_disagree": 0.169921875899 },900 {901 "tau": 2.625,902 "escalated_frac": 0.8385416666666666,903 "residual_disagree": 0.15234375904 },905 {906 "tau": 2.625,907 "escalated_frac": 0.8385416666666666,908 "residual_disagree": 0.15234375909 },910 {911 "tau": 2.625,912 "escalated_frac": 0.8385416666666666,913 "residual_disagree": 0.15234375914 },915 {916 "tau": 2.75,917 "escalated_frac": 0.853515625,918 "residual_disagree": 0.13736979166666666919 },920 {921 "tau": 2.75,922 "escalated_frac": 0.853515625,923 "residual_disagree": 0.13736979166666666924 },925 {926 "tau": 2.75,927 "escalated_frac": 0.853515625,928 "residual_disagree": 0.13736979166666666929 },930 {931 "tau": 2.875,932 "escalated_frac": 0.86669921875,933 "residual_disagree": 0.12483723958333333934 },935 {936 "tau": 2.875,937 "escalated_frac": 0.86669921875,938 "residual_disagree": 0.12483723958333333939 },940 {941 "tau": 2.875,942 "escalated_frac": 0.86669921875,943 "residual_disagree": 0.12483723958333333944 },945 {946 "tau": 3.0,947 "escalated_frac": 0.88134765625,948 "residual_disagree": 0.11051432291666667949 },950 {951 "tau": 3.0,952 "escalated_frac": 0.88134765625,953 "residual_disagree": 0.11051432291666667954 },955 {956 "tau": 3.125,957 "escalated_frac": 0.8942057291666666,958 "residual_disagree": 0.09814453125959 },960 {961 "tau": 3.125,962 "escalated_frac": 0.8942057291666666,963 "residual_disagree": 0.09814453125964 },965 {966 "tau": 3.217650753768851,967 "escalated_frac": 0.9044596354166666,968 "residual_disagree": 0.08837890625969 },970 {971 "tau": 3.25,972 "escalated_frac": 0.9044596354166666,973 "residual_disagree": 0.08837890625974 },975 {976 "tau": 3.375,977 "escalated_frac": 0.9130859375,978 "residual_disagree": 0.08024088541666667979 },980 {981 "tau": 3.375,982 "escalated_frac": 0.9130859375,983 "residual_disagree": 0.08024088541666667984 },985 {986 "tau": 3.5,987 "escalated_frac": 0.92041015625,988 "residual_disagree": 0.07307942708333333989 },990 {991 "tau": 3.625,992 "escalated_frac": 0.9267578125,993 "residual_disagree": 0.06722005208333333994 },995 {996 "tau": 3.75,997 "escalated_frac": 0.93310546875,998 "residual_disagree": 0.0615234375999 },1000 {1001 "tau": 3.75,1002 "escalated_frac": 0.93310546875,1003 "residual_disagree": 0.06152343751004 },1005 {1006 "tau": 3.875,1007 "escalated_frac": 0.9401041666666666,1008 "residual_disagree": 0.0548502604166666641009 },1010 {1011 "tau": 4.0,1012 "escalated_frac": 0.9456380208333334,1013 "residual_disagree": 0.0496419270833333361014 },1015 {1016 "tau": 4.125,1017 "escalated_frac": 0.95166015625,1018 "residual_disagree": 0.04394531251019 },1020 {1021 "tau": 4.25,1022 "escalated_frac": 0.9560546875,1023 "residual_disagree": 0.0398763020833333361024 },1025 {1026 "tau": 4.5,1027 "escalated_frac": 0.9637044270833334,1028 "residual_disagree": 0.0325520833333333361029 },1030 {1031 "tau": 4.75,1032 "escalated_frac": 0.9690755208333334,1033 "residual_disagree": 0.027343751034 },1035 {1036 "tau": 5.0,1037 "escalated_frac": 0.974609375,1038 "residual_disagree": 0.0222981770833333321039 },1040 {1041 "tau": 5.25,1042 "escalated_frac": 0.97802734375,1043 "residual_disagree": 0.019042968751044 },1045 {1046 "tau": 5.75,1047 "escalated_frac": 0.9847005208333334,1048 "residual_disagree": 0.013183593751049 },1050 {1051 "tau": 6.25,1052 "escalated_frac": 0.98974609375,1053 "residual_disagree": 0.0084635416666666661054 },1055 {1056 "tau": 7.25,1057 "escalated_frac": 0.9947916666666666,1058 "residual_disagree": 0.0040690104166666671059 },1060 {1061 "tau": 14.875,1062 "escalated_frac": 0.9998372395833334,1063 "residual_disagree": 0.000162760416666666661064 }1065 ],1066 "per_domain": {1067 "code": {1068 "agreement_rate": 0.017578125,1069 "auroc": 0.56069140711287821070 },1071 "math": {1072 "agreement_rate": 0.0546875,1073 "auroc": 0.65184105667060211074 },1075 "chat": {1076 "agreement_rate": 0.0205078125,1077 "auroc": 0.55521530646156771078 },1079 "french": {1080 "agreement_rate": 0.0263671875,1081 "auroc": 0.61003380511906091082 },1083 "english": {1084 "agreement_rate": 0.029296875,1085 "auroc": 0.64285714285714291086 },1087 "reasoning": {1088 "agreement_rate": 0.029296875,1089 "auroc": 0.53584842387659291090 }1091 },1092 "escalation_frac_for_99pct": 0.989746093751093 },1094 "3": {1095 "bits": 3,1096 "n_positions": 6144,1097 "agreement_rate": 0.73095703125,1098 "mean_kl_ref_q": 0.7811363339424133,1099 "auroc_margin_predicts_disagreement": 0.8521382349292252,1100 "median_margin_agree": 5.25,1101 "median_margin_disagree": 1.125,1102 "escalation_curve": [1103 {1104 "tau": 0.0,1105 "escalated_frac": 0.0,1106 "residual_disagree": 0.269042968751107 },1108 {1109 "tau": 0.0,1110 "escalated_frac": 0.0,1111 "residual_disagree": 0.269042968751112 },1113 {1114 "tau": 0.0,1115 "escalated_frac": 0.0,1116 "residual_disagree": 0.269042968751117 },1118 {1119 "tau": 0.0,1120 "escalated_frac": 0.0,1121 "residual_disagree": 0.269042968751122 },1123 {1124 "tau": 0.0,1125 "escalated_frac": 0.0,1126 "residual_disagree": 0.269042968751127 },1128 {1129 "tau": 0.125,1130 "escalated_frac": 0.020833333333333332,1131 "residual_disagree": 0.25504557291666671132 },1133 {1134 "tau": 0.125,1135 "escalated_frac": 0.020833333333333332,1136 "residual_disagree": 0.25504557291666671137 },1138 {1139 "tau": 0.125,1140 "escalated_frac": 0.020833333333333332,1141 "residual_disagree": 0.25504557291666671142 },1143 {1144 "tau": 0.25,1145 "escalated_frac": 0.038248697916666664,1146 "residual_disagree": 0.24218751147 },1148 {1149 "tau": 0.25,1150 "escalated_frac": 0.038248697916666664,1151 "residual_disagree": 0.24218751152 },1153 {1154 "tau": 0.25,1155 "escalated_frac": 0.038248697916666664,1156 "residual_disagree": 0.24218751157 },1158 {1159 "tau": 0.25,1160 "escalated_frac": 0.038248697916666664,1161 "residual_disagree": 0.24218751162 },1163 {1164 "tau": 0.25,1165 "escalated_frac": 0.038248697916666664,1166 "residual_disagree": 0.24218751167 },1168 {1169 "tau": 0.25,1170 "escalated_frac": 0.038248697916666664,1171 "residual_disagree": 0.24218751172 },1173 {1174 "tau": 0.25,1175 "escalated_frac": 0.038248697916666664,1176 "residual_disagree": 0.24218751177 },1178 {1179 "tau": 0.375,1180 "escalated_frac": 0.07096354166666667,1181 "residual_disagree": 0.219889322916666661182 },1183 {1184 "tau": 0.375,1185 "escalated_frac": 0.07096354166666667,1186 "residual_disagree": 0.219889322916666661187 },1188 {1189 "tau": 0.375,1190 "escalated_frac": 0.07096354166666667,1191 "residual_disagree": 0.219889322916666661192 },1193 {1194 "tau": 0.5,1195 "escalated_frac": 0.08723958333333333,1196 "residual_disagree": 0.209309895833333341197 },1198 {1199 "tau": 0.5,1200 "escalated_frac": 0.08723958333333333,1201 "residual_disagree": 0.209309895833333341202 },1203 {1204 "tau": 0.5,1205 "escalated_frac": 0.08723958333333333,1206 "residual_disagree": 0.209309895833333341207 },1208 {1209 "tau": 0.5,1210 "escalated_frac": 0.08723958333333333,1211 "residual_disagree": 0.209309895833333341212 },1213 {1214 "tau": 0.5,1215 "escalated_frac": 0.08723958333333333,1216 "residual_disagree": 0.209309895833333341217 },1218 {1219 "tau": 0.5,1220 "escalated_frac": 0.08723958333333333,1221 "residual_disagree": 0.209309895833333341222 },1223 {1224 "tau": 0.625,1225 "escalated_frac": 0.11946614583333333,1226 "residual_disagree": 0.19042968751227 },1228 {1229 "tau": 0.625,1230 "escalated_frac": 0.11946614583333333,1231 "residual_disagree": 0.19042968751232 },1233 {1234 "tau": 0.625,1235 "escalated_frac": 0.11946614583333333,1236 "residual_disagree": 0.19042968751237 },1238 {1239 "tau": 0.625,1240 "escalated_frac": 0.11946614583333333,1241 "residual_disagree": 0.19042968751242 },1243 {1244 "tau": 0.75,1245 "escalated_frac": 0.13688151041666666,1246 "residual_disagree": 0.180013020833333341247 },1248 {1249 "tau": 0.75,1250 "escalated_frac": 0.13688151041666666,1251 "residual_disagree": 0.180013020833333341252 },1253 {1254 "tau": 0.75,1255 "escalated_frac": 0.13688151041666666,1256 "residual_disagree": 0.180013020833333341257 },1258 {1259 "tau": 0.75,1260 "escalated_frac": 0.13688151041666666,1261 "residual_disagree": 0.180013020833333341262 },1263 {1264 "tau": 0.75,1265 "escalated_frac": 0.13688151041666666,1266 "residual_disagree": 0.180013020833333341267 },1268 {1269 "tau": 0.875,1270 "escalated_frac": 0.16552734375,1271 "residual_disagree": 0.161621093751272 },1273 {1274 "tau": 0.875,1275 "escalated_frac": 0.16552734375,1276 "residual_disagree": 0.161621093751277 },1278 {1279 "tau": 0.875,1280 "escalated_frac": 0.16552734375,1281 "residual_disagree": 0.161621093751282 },1283 {1284 "tau": 0.875,1285 "escalated_frac": 0.16552734375,1286 "residual_disagree": 0.161621093751287 },1288 {1289 "tau": 1.0,1290 "escalated_frac": 0.18131510416666666,1291 "residual_disagree": 0.151855468751292 },1293 {1294 "tau": 1.0,1295 "escalated_frac": 0.18131510416666666,1296 "residual_disagree": 0.151855468751297 },1298 {1299 "tau": 1.0,1300 "escalated_frac": 0.18131510416666666,1301 "residual_disagree": 0.151855468751302 },1303 {1304 "tau": 1.0,1305 "escalated_frac": 0.18131510416666666,1306 "residual_disagree": 0.151855468751307 },1308 {1309 "tau": 1.0,1310 "escalated_frac": 0.18131510416666666,1311 "residual_disagree": 0.151855468751312 },1313 {1314 "tau": 1.0,1315 "escalated_frac": 0.18131510416666666,1316 "residual_disagree": 0.151855468751317 },1318 {1319 "tau": 1.125,1320 "escalated_frac": 0.21223958333333334,1321 "residual_disagree": 0.135253906251322 },1323 {1324 "tau": 1.125,1325 "escalated_frac": 0.21223958333333334,1326 "residual_disagree": 0.135253906251327 },1328 {1329 "tau": 1.25,1330 "escalated_frac": 0.22574869791666666,1331 "residual_disagree": 0.128906251332 },1333 {1334 "tau": 1.25,1335 "escalated_frac": 0.22574869791666666,1336 "residual_disagree": 0.128906251337 },1338 {1339 "tau": 1.25,1340 "escalated_frac": 0.22574869791666666,1341 "residual_disagree": 0.128906251342 },1343 {1344 "tau": 1.25,1345 "escalated_frac": 0.22574869791666666,1346 "residual_disagree": 0.128906251347 },1348 {1349 "tau": 1.25,1350 "escalated_frac": 0.22574869791666666,1351 "residual_disagree": 0.128906251352 },1353 {1354 "tau": 1.25,1355 "escalated_frac": 0.22574869791666666,1356 "residual_disagree": 0.128906251357 },1358 {1359 "tau": 1.375,1360 "escalated_frac": 0.2521158854166667,1361 "residual_disagree": 0.115559895833333331362 },1363 {1364 "tau": 1.375,1365 "escalated_frac": 0.2521158854166667,1366 "residual_disagree": 0.115559895833333331367 },1368 {1369 "tau": 1.5,1370 "escalated_frac": 0.2635091145833333,1371 "residual_disagree": 0.108072916666666671372 },1373 {1374 "tau": 1.5,1375 "escalated_frac": 0.2635091145833333,1376 "residual_disagree": 0.108072916666666671377 },1378 {1379 "tau": 1.5,1380 "escalated_frac": 0.2635091145833333,1381 "residual_disagree": 0.108072916666666671382 },1383 {1384 "tau": 1.5,1385 "escalated_frac": 0.2635091145833333,1386 "residual_disagree": 0.108072916666666671387 },1388 {1389 "tau": 1.625,1390 "escalated_frac": 0.28466796875,1391 "residual_disagree": 0.097330729166666671392 },1393 {1394 "tau": 1.625,1395 "escalated_frac": 0.28466796875,1396 "residual_disagree": 0.097330729166666671397 },1398 {1399 "tau": 1.625,1400 "escalated_frac": 0.28466796875,1401 "residual_disagree": 0.097330729166666671402 },1403 {1404 "tau": 1.75,1405 "escalated_frac": 0.2985026041666667,1406 "residual_disagree": 0.090006510416666671407 },1408 {1409 "tau": 1.75,1410 "escalated_frac": 0.2985026041666667,1411 "residual_disagree": 0.090006510416666671412 },1413 {1414 "tau": 1.75,1415 "escalated_frac": 0.2985026041666667,1416 "residual_disagree": 0.090006510416666671417 },1418 {1419 "tau": 1.75,1420 "escalated_frac": 0.2985026041666667,1421 "residual_disagree": 0.090006510416666671422 },1423 {1424 "tau": 1.75,1425 "escalated_frac": 0.2985026041666667,1426 "residual_disagree": 0.090006510416666671427 },1428 {1429 "tau": 1.875,1430 "escalated_frac": 0.32177734375,1431 "residual_disagree": 0.079264322916666671432 },1433 {1434 "tau": 1.875,1435 "escalated_frac": 0.32177734375,1436 "residual_disagree": 0.079264322916666671437 },1438 {1439 "tau": 2.0,1440 "escalated_frac": 0.33203125,1441 "residual_disagree": 0.07519531251442 },1443 {1444 "tau": 2.0,1445 "escalated_frac": 0.33203125,1446 "residual_disagree": 0.07519531251447 },1448 {1449 "tau": 2.0,1450 "escalated_frac": 0.33203125,1451 "residual_disagree": 0.07519531251452 },1453 {1454 "tau": 2.0,1455 "escalated_frac": 0.33203125,1456 "residual_disagree": 0.07519531251457 },1458 {1459 "tau": 2.125,1460 "escalated_frac": 0.3527018229166667,1461 "residual_disagree": 0.0683593751462 },1463 {1464 "tau": 2.125,1465 "escalated_frac": 0.3527018229166667,1466 "residual_disagree": 0.0683593751467 },1468 {1469 "tau": 2.25,1470 "escalated_frac": 0.3619791666666667,1471 "residual_disagree": 0.064615885416666671472 },1473 {1474 "tau": 2.25,1475 "escalated_frac": 0.3619791666666667,1476 "residual_disagree": 0.064615885416666671477 },1478 {1479 "tau": 2.25,1480 "escalated_frac": 0.3619791666666667,1481 "residual_disagree": 0.064615885416666671482 },1483 {1484 "tau": 2.375,1485 "escalated_frac": 0.3811848958333333,1486 "residual_disagree": 0.0579427083333333361487 },1488 {1489 "tau": 2.375,1490 "escalated_frac": 0.3811848958333333,1491 "residual_disagree": 0.0579427083333333361492 },1493 {1494 "tau": 2.5,1495 "escalated_frac": 0.3907877604166667,1496 "residual_disagree": 0.05468751497 },1498 {1499 "tau": 2.5,1500 "escalated_frac": 0.3907877604166667,1501 "residual_disagree": 0.05468751502 },1503 {1504 "tau": 2.5,1505 "escalated_frac": 0.3907877604166667,1506 "residual_disagree": 0.05468751507 },1508 {1509 "tau": 2.5,1510 "escalated_frac": 0.3907877604166667,1511 "residual_disagree": 0.05468751512 },1513 {1514 "tau": 2.625,1515 "escalated_frac": 0.4083658854166667,1516 "residual_disagree": 0.049316406251517 },1518 {1519 "tau": 2.75,1520 "escalated_frac": 0.41650390625,1521 "residual_disagree": 0.0465494791666666641522 },1523 {1524 "tau": 2.75,1525 "escalated_frac": 0.41650390625,1526 "residual_disagree": 0.0465494791666666641527 },1528 {1529 "tau": 2.75,1530 "escalated_frac": 0.41650390625,1531 "residual_disagree": 0.0465494791666666641532 },1533 {1534 "tau": 2.75,1535 "escalated_frac": 0.41650390625,1536 "residual_disagree": 0.0465494791666666641537 },1538 {1539 "tau": 2.875,1540 "escalated_frac": 0.4334309895833333,1541 "residual_disagree": 0.0402018229166666641542 },1543 {1544 "tau": 3.0,1545 "escalated_frac": 0.4404296875,1546 "residual_disagree": 0.0382486979166666641547 },1548 {1549 "tau": 3.0,1550 "escalated_frac": 0.4404296875,1551 "residual_disagree": 0.0382486979166666641552 },1553 {1554 "tau": 3.0,1555 "escalated_frac": 0.4404296875,1556 "residual_disagree": 0.0382486979166666641557 },1558 {1559 "tau": 3.0,1560 "escalated_frac": 0.4404296875,1561 "residual_disagree": 0.0382486979166666641562 },1563 {1564 "tau": 3.125,1565 "escalated_frac": 0.4591471354166667,1566 "residual_disagree": 0.0340169270833333361567 },1568 {1569 "tau": 3.125,1570 "escalated_frac": 0.4591471354166667,1571 "residual_disagree": 0.0340169270833333361572 },1573 {1574 "tau": 3.25,1575 "escalated_frac": 0.4685872395833333,1576 "residual_disagree": 0.0315755208333333361577 },1578 {1579 "tau": 3.25,1580 "escalated_frac": 0.4685872395833333,1581 "residual_disagree": 0.0315755208333333361582 },1583 {1584 "tau": 3.25,1585 "escalated_frac": 0.4685872395833333,1586 "residual_disagree": 0.0315755208333333361587 },1588 {1589 "tau": 3.375,1590 "escalated_frac": 0.4855143229166667,1591 "residual_disagree": 0.0265299479166666681592 },1593 {1594 "tau": 3.375,1595 "escalated_frac": 0.4855143229166667,1596 "residual_disagree": 0.0265299479166666681597 },1598 {1599 "tau": 3.5,1600 "escalated_frac": 0.4947916666666667,1601 "residual_disagree": 0.0242513020833333321602 },1603 {1604 "tau": 3.5,1605 "escalated_frac": 0.4947916666666667,1606 "residual_disagree": 0.0242513020833333321607 },1608 {1609 "tau": 3.5,1610 "escalated_frac": 0.4947916666666667,1611 "residual_disagree": 0.0242513020833333321612 },1613 {1614 "tau": 3.625,1615 "escalated_frac": 0.5110677083333334,1616 "residual_disagree": 0.0213216145833333321617 },1618 {1619 "tau": 3.75,1620 "escalated_frac": 0.51708984375,1621 "residual_disagree": 0.020019531251622 },1623 {1624 "tau": 3.75,1625 "escalated_frac": 0.51708984375,1626 "residual_disagree": 0.020019531251627 },1628 {1629 "tau": 3.75,1630 "escalated_frac": 0.51708984375,1631 "residual_disagree": 0.020019531251632 },1633 {1634 "tau": 3.875,1635 "escalated_frac": 0.53173828125,1636 "residual_disagree": 0.0174153645833333321637 },1638 {1639 "tau": 3.875,1640 "escalated_frac": 0.53173828125,1641 "residual_disagree": 0.0174153645833333321642 },1643 {1644 "tau": 4.0,1645 "escalated_frac": 0.5384114583333334,1646 "residual_disagree": 0.0169270833333333321647 },1648 {1649 "tau": 4.0,1650 "escalated_frac": 0.5384114583333334,1651 "residual_disagree": 0.0169270833333333321652 },1653 {1654 "tau": 4.125,1655 "escalated_frac": 0.5511067708333334,1656 "residual_disagree": 0.014160156251657 },1658 {1659 "tau": 4.25,1660 "escalated_frac": 0.5568033854166666,1661 "residual_disagree": 0.0139973958333333341662 },1663 {1664 "tau": 4.25,1665 "escalated_frac": 0.5568033854166666,1666 "residual_disagree": 0.0139973958333333341667 },1668 {1669 "tau": 4.25,1670 "escalated_frac": 0.5568033854166666,1671 "residual_disagree": 0.0139973958333333341672 },1673 {1674 "tau": 4.375,1675 "escalated_frac": 0.5690104166666666,1676 "residual_disagree": 0.013183593751677 },1678 {1679 "tau": 4.5,1680 "escalated_frac": 0.5745442708333334,1681 "residual_disagree": 0.0128580729166666661682 },1683 {1684 "tau": 4.5,1685 "escalated_frac": 0.5745442708333334,1686 "residual_disagree": 0.0128580729166666661687 },1688 {1689 "tau": 4.625,1690 "escalated_frac": 0.5865885416666666,1691 "residual_disagree": 0.0113932291666666661692 },1693 {1694 "tau": 4.75,1695 "escalated_frac": 0.5926106770833334,1696 "residual_disagree": 0.0109049479166666661697 },1698 {1699 "tau": 4.75,1700 "escalated_frac": 0.5926106770833334,1701 "residual_disagree": 0.0109049479166666661702 },1703 {1704 "tau": 4.875,1705 "escalated_frac": 0.6017252604166666,1706 "residual_disagree": 0.0099283854166666661707 },1708 {1709 "tau": 5.0,1710 "escalated_frac": 0.6067708333333334,1711 "residual_disagree": 0.009277343751712 },1713 {1714 "tau": 5.0,1715 "escalated_frac": 0.6067708333333334,1716 "residual_disagree": 0.009277343751717 },1718 {1719 "tau": 5.125,1720 "escalated_frac": 0.61767578125,1721 "residual_disagree": 0.00878906251722 },1723 {1724 "tau": 5.25,1725 "escalated_frac": 0.6214192708333334,1726 "residual_disagree": 0.0086263020833333341727 },1728 {1729 "tau": 5.25,1730 "escalated_frac": 0.6214192708333334,1731 "residual_disagree": 0.0086263020833333341732 },1733 {1734 "tau": 5.375,1735 "escalated_frac": 0.63134765625,1736 "residual_disagree": 0.007324218751737 },1738 {1739 "tau": 5.5,1740 "escalated_frac": 0.6359049479166666,1741 "residual_disagree": 0.00683593751742 },1743 {1744 "tau": 5.5,1745 "escalated_frac": 0.6359049479166666,1746 "residual_disagree": 0.00683593751747 },1748 {1749 "tau": 5.75,1750 "escalated_frac": 0.6466471354166666,1751 "residual_disagree": 0.0065104166666666671752 },1753 {1754 "tau": 5.75,1755 "escalated_frac": 0.6466471354166666,1756 "residual_disagree": 0.0065104166666666671757 },1758 {1759 "tau": 6.0,1760 "escalated_frac": 0.6573893229166666,1761 "residual_disagree": 0.006347656251762 },1763 {1764 "tau": 6.0,1765 "escalated_frac": 0.6573893229166666,1766 "residual_disagree": 0.006347656251767 },1768 {1769 "tau": 6.25,1770 "escalated_frac": 0.6661783854166666,1771 "residual_disagree": 0.0052083333333333331772 },1773 {1774 "tau": 6.375,1775 "escalated_frac": 0.67236328125,1776 "residual_disagree": 0.0047200520833333331777 },1778 {1779 "tau": 6.5,1780 "escalated_frac": 0.6748046875,1781 "residual_disagree": 0.0047200520833333331782 },1783 {1784 "tau": 6.625,1785 "escalated_frac": 0.6818033854166666,1786 "residual_disagree": 0.0040690104166666671787 },1788 {1789 "tau": 6.75,1790 "escalated_frac": 0.6863606770833334,1791 "residual_disagree": 0.003906251792 },1793 {1794 "tau": 6.871231155778901,1795 "escalated_frac": 0.693359375,1796 "residual_disagree": 0.00358072916666666651797 },1798 {1799 "tau": 7.0,1800 "escalated_frac": 0.6964518229166666,1801 "residual_disagree": 0.00358072916666666651802 },1803 {1804 "tau": 7.125,1805 "escalated_frac": 0.7024739583333334,1806 "residual_disagree": 0.00325520833333333351807 },1808 {1809 "tau": 7.25,1810 "escalated_frac": 0.7052408854166666,1811 "residual_disagree": 0.00309244791666666651812 },1813 {1814 "tau": 7.5,1815 "escalated_frac": 0.71337890625,1816 "residual_disagree": 0.00276692708333333351817 },1818 {1819 "tau": 7.5,1820 "escalated_frac": 0.71337890625,1821 "residual_disagree": 0.00276692708333333351822 },1823 {1824 "tau": 7.75,1825 "escalated_frac": 0.7233072916666666,1826 "residual_disagree": 0.002441406251827 },1828 {1829 "tau": 7.75,1830 "escalated_frac": 0.7233072916666666,1831 "residual_disagree": 0.002441406251832 },1833 {1834 "tau": 7.875,1835 "escalated_frac": 0.7314453125,1836 "residual_disagree": 0.00227864583333333351837 },1838 {1839 "tau": 8.125,1840 "escalated_frac": 0.7373046875,1841 "residual_disagree": 0.00227864583333333351842 },1843 {1844 "tau": 8.25,1845 "escalated_frac": 0.73974609375,1846 "residual_disagree": 0.00227864583333333351847 },1848 {1849 "tau": 8.5,1850 "escalated_frac": 0.7473958333333334,1851 "residual_disagree": 0.00211588541666666651852 },1853 {1854 "tau": 8.75,1855 "escalated_frac": 0.7535807291666666,1856 "residual_disagree": 0.00179036458333333331857 },1858 {1859 "tau": 8.875,1860 "escalated_frac": 0.7576497395833334,1861 "residual_disagree": 0.00179036458333333331862 },1863 {1864 "tau": 9.0,1865 "escalated_frac": 0.7600911458333334,1866 "residual_disagree": 0.00179036458333333331867 },1868 {1869 "tau": 9.25,1870 "escalated_frac": 0.7669270833333334,1871 "residual_disagree": 0.00179036458333333331872 },1873 {1874 "tau": 9.5,1875 "escalated_frac": 0.7736002604166666,1876 "residual_disagree": 0.00179036458333333331877 },1878 {1879 "tau": 9.5,1880 "escalated_frac": 0.7736002604166666,1881 "residual_disagree": 0.00179036458333333331882 },1883 {1884 "tau": 9.75,1885 "escalated_frac": 0.78076171875,1886 "residual_disagree": 0.00162760416666666671887 },1888 {1889 "tau": 10.0,1890 "escalated_frac": 0.7862955729166666,1891 "residual_disagree": 0.001464843751892 },1893 {1894 "tau": 10.25,1895 "escalated_frac": 0.7926432291666666,1896 "residual_disagree": 0.001464843751897 },1898 {1899 "tau": 10.375,1900 "escalated_frac": 0.7967122395833334,1901 "residual_disagree": 0.00130208333333333331902 },1903 {1904 "tau": 10.625,1905 "escalated_frac": 0.8035481770833334,1906 "residual_disagree": 0.00130208333333333331907 },1908 {1909 "tau": 10.75,1910 "escalated_frac": 0.8059895833333334,1911 "residual_disagree": 0.00130208333333333331912 },1913 {1914 "tau": 10.979271356783897,1915 "escalated_frac": 0.81396484375,1916 "residual_disagree": 0.00097656251917 },1918 {1919 "tau": 11.125,1920 "escalated_frac": 0.81787109375,1921 "residual_disagree": 0.00097656251922 },1923 {1924 "tau": 11.25,1925 "escalated_frac": 0.8203125,1926 "residual_disagree": 0.00097656251927 },1928 {1929 "tau": 11.5,1930 "escalated_frac": 0.8269856770833334,1931 "residual_disagree": 0.00097656251932 },1933 {1934 "tau": 11.75,1935 "escalated_frac": 0.8336588541666666,1936 "residual_disagree": 0.00097656251937 },1938 {1939 "tau": 11.875,1940 "escalated_frac": 0.8369140625,1941 "residual_disagree": 0.00097656251942 },1943 {1944 "tau": 12.125,1945 "escalated_frac": 0.8430989583333334,1946 "residual_disagree": 0.00081380208333333341947 },1948 {1949 "tau": 12.364949748743697,1950 "escalated_frac": 0.84912109375,1951 "residual_disagree": 0.00065104166666666661952 },1953 {1954 "tau": 12.5,1955 "escalated_frac": 0.8518880208333334,1956 "residual_disagree": 0.00065104166666666661957 },1958 {1959 "tau": 12.75,1960 "escalated_frac": 0.8583984375,1961 "residual_disagree": 0.000488281251962 },1963 {1964 "tau": 13.0,1965 "escalated_frac": 0.8640950520833334,1966 "residual_disagree": 0.00032552083333333331967 },1968 {1969 "tau": 13.25,1970 "escalated_frac": 0.8684895833333334,1971 "residual_disagree": 0.00032552083333333331972 },1973 {1974 "tau": 13.5,1975 "escalated_frac": 0.87353515625,1976 "residual_disagree": 0.000162760416666666661977 },1978 {1979 "tau": 13.75,1980 "escalated_frac": 0.8790690104166666,1981 "residual_disagree": 0.01982 },1983 {1984 "tau": 13.875,1985 "escalated_frac": 0.8834635416666666,1986 "residual_disagree": 0.01987 },1988 {1989 "tau": 14.125,1990 "escalated_frac": 0.8885091145833334,1991 "residual_disagree": 0.01992 },1993 {1994 "tau": 14.25,1995 "escalated_frac": 0.8917643229166666,1996 "residual_disagree": 0.01997 },1998 {1999 "tau": 14.5,2000 "escalated_frac": 0.8963216145833334,2001 "residual_disagree": 0.02002 },2003 {2004 "tau": 14.75,2005 "escalated_frac": 0.9021809895833334,2006 "residual_disagree": 0.02007 },2008 {2009 "tau": 15.0,2010 "escalated_frac": 0.9075520833333334,2011 "residual_disagree": 0.02012 },2013 {2014 "tau": 15.27763819095469,2015 "escalated_frac": 0.91455078125,2016 "residual_disagree": 0.02017 },2018 {2019 "tau": 15.625,2020 "escalated_frac": 0.9189453125,2021 "residual_disagree": 0.02022 },2023 {2024 "tau": 16.0,2025 "escalated_frac": 0.9236653645833334,2026 "residual_disagree": 0.02027 },2028 {2029 "tau": 16.25,2030 "escalated_frac": 0.92822265625,2031 "residual_disagree": 0.02032 },2033 {2034 "tau": 16.75,2035 "escalated_frac": 0.9339192708333334,2036 "residual_disagree": 0.02037 },2038 {2039 "tau": 17.25,2040 "escalated_frac": 0.9392903645833334,2041 "residual_disagree": 0.02042 },2043 {2044 "tau": 17.625,2045 "escalated_frac": 0.9440104166666666,2046 "residual_disagree": 0.02047 },2048 {2049 "tau": 17.875,2050 "escalated_frac": 0.9484049479166666,2051 "residual_disagree": 0.02052 },2053 {2054 "tau": 18.375,2055 "escalated_frac": 0.9542643229166666,2056 "residual_disagree": 0.02057 },2058 {2059 "tau": 18.875,2060 "escalated_frac": 0.95947265625,2061 "residual_disagree": 0.02062 },2063 {2064 "tau": 19.375,2065 "escalated_frac": 0.9640299479166666,2066 "residual_disagree": 0.02067 },2068 {2069 "tau": 20.0,2070 "escalated_frac": 0.9695638020833334,2071 "residual_disagree": 0.02072 },2073 {2074 "tau": 20.375,2075 "escalated_frac": 0.97412109375,2076 "residual_disagree": 0.02077 },2078 {2079 "tau": 21.125,2080 "escalated_frac": 0.9793294270833334,2081 "residual_disagree": 0.02082 },2083 {2084 "tau": 22.173994974874404,2085 "escalated_frac": 0.98486328125,2086 "residual_disagree": 0.02087 },2088 {2089 "tau": 23.53266331658301,2090 "escalated_frac": 0.9899088541666666,2091 "residual_disagree": 0.02092 },2093 {2094 "tau": 25.391331658291392,2095 "escalated_frac": 0.9949544270833334,2096 "residual_disagree": 0.02097 },2098 {2099 "tau": 32.375,2100 "escalated_frac": 0.9998372395833334,2101 "residual_disagree": 0.02102 }2103 ],2104 "per_domain": {2105 "code": {2106 "agreement_rate": 0.728515625,2107 "auroc": 0.83754845989160412108 },2109 "math": {2110 "agreement_rate": 0.8193359375,2111 "auroc": 0.89571884160680352112 },2113 "chat": {2114 "agreement_rate": 0.6669921875,2115 "auroc": 0.80092570726869132116 },2117 "french": {2118 "agreement_rate": 0.728515625,2119 "auroc": 0.84656055316604632120 },2121 "english": {2122 "agreement_rate": 0.6875,2123 "auroc": 0.85438343394886362124 },2125 "reasoning": {2126 "agreement_rate": 0.7548828125,2127 "auroc": 0.86268380552821062128 }2129 },2130 "escalation_frac_for_99pct": 0.60172526041666662131 },2132 "4": {2133 "bits": 4,2134 "n_positions": 6144,2135 "agreement_rate": 0.8740234375,2136 "mean_kl_ref_q": 0.19848966598510742,2137 "auroc_margin_predicts_disagreement": 0.8982447466304814,2138 "median_margin_agree": 6.5,2139 "median_margin_disagree": 0.75,2140 "escalation_curve": [2141 {2142 "tau": 0.0,2143 "escalated_frac": 0.0,2144 "residual_disagree": 0.12597656252145 },2146 {2147 "tau": 0.0,2148 "escalated_frac": 0.0,2149 "residual_disagree": 0.12597656252150 },2151 {2152 "tau": 0.0,2153 "escalated_frac": 0.0,2154 "residual_disagree": 0.12597656252155 },2156 {2157 "tau": 0.125,2158 "escalated_frac": 0.014485677083333334,2159 "residual_disagree": 0.117838541666666672160 },2161 {2162 "tau": 0.125,2163 "escalated_frac": 0.014485677083333334,2164 "residual_disagree": 0.117838541666666672165 },2166 {2167 "tau": 0.125,2168 "escalated_frac": 0.014485677083333334,2169 "residual_disagree": 0.117838541666666672170 },2171 {2172 "tau": 0.25,2173 "escalated_frac": 0.0263671875,2174 "residual_disagree": 0.10839843752175 },2176 {2177 "tau": 0.25,2178 "escalated_frac": 0.0263671875,2179 "residual_disagree": 0.10839843752180 },2181 {2182 "tau": 0.25,2183 "escalated_frac": 0.0263671875,2184 "residual_disagree": 0.10839843752185 },2186 {2187 "tau": 0.25,2188 "escalated_frac": 0.0263671875,2189 "residual_disagree": 0.10839843752190 },2191 {2192 "tau": 0.25,2193 "escalated_frac": 0.0263671875,2194 "residual_disagree": 0.10839843752195 },2196 {2197 "tau": 0.375,2198 "escalated_frac": 0.053059895833333336,2199 "residual_disagree": 0.093098958333333332200 },2201 {2202 "tau": 0.375,2203 "escalated_frac": 0.053059895833333336,2204 "residual_disagree": 0.093098958333333332205 },2206 {2207 "tau": 0.5,2208 "escalated_frac": 0.06266276041666667,2209 "residual_disagree": 0.087565104166666672210 },2211 {2212 "tau": 0.5,2213 "escalated_frac": 0.06266276041666667,2214 "residual_disagree": 0.087565104166666672215 },2216 {2217 "tau": 0.5,2218 "escalated_frac": 0.06266276041666667,2219 "residual_disagree": 0.087565104166666672220 },2221 {2222 "tau": 0.5,2223 "escalated_frac": 0.06266276041666667,2224 "residual_disagree": 0.087565104166666672225 },2226 {2227 "tau": 0.5,2228 "escalated_frac": 0.06266276041666667,2229 "residual_disagree": 0.087565104166666672230 },2231 {2232 "tau": 0.625,2233 "escalated_frac": 0.08821614583333333,2234 "residual_disagree": 0.076822916666666672235 },2236 {2237 "tau": 0.625,2238 "escalated_frac": 0.08821614583333333,2239 "residual_disagree": 0.076822916666666672240 },2241 {2242 "tau": 0.75,2243 "escalated_frac": 0.09781901041666667,2244 "residual_disagree": 0.072916666666666672245 },2246 {2247 "tau": 0.75,2248 "escalated_frac": 0.09781901041666667,2249 "residual_disagree": 0.072916666666666672250 },2251 {2252 "tau": 0.75,2253 "escalated_frac": 0.09781901041666667,2254 "residual_disagree": 0.072916666666666672255 },2256 {2257 "tau": 0.75,2258 "escalated_frac": 0.09781901041666667,2259 "residual_disagree": 0.072916666666666672260 },2261 {2262 "tau": 0.75,2263 "escalated_frac": 0.09781901041666667,2264 "residual_disagree": 0.072916666666666672265 },2266 {2267 "tau": 0.875,2268 "escalated_frac": 0.12337239583333333,2269 "residual_disagree": 0.0621744791666666642270 },2271 {2272 "tau": 0.875,2273 "escalated_frac": 0.12337239583333333,2274 "residual_disagree": 0.0621744791666666642275 },2276 {2277 "tau": 1.0,2278 "escalated_frac": 0.1318359375,2279 "residual_disagree": 0.058593752280 },2281 {2282 "tau": 1.0,2283 "escalated_frac": 0.1318359375,2284 "residual_disagree": 0.058593752285 },2286 {2287 "tau": 1.0,2288 "escalated_frac": 0.1318359375,2289 "residual_disagree": 0.058593752290 },2291 {2292 "tau": 1.0,2293 "escalated_frac": 0.1318359375,2294 "residual_disagree": 0.058593752295 },2296 {2297 "tau": 1.125,2298 "escalated_frac": 0.15543619791666666,2299 "residual_disagree": 0.0504557291666666642300 },2301 {2302 "tau": 1.125,2303 "escalated_frac": 0.15543619791666666,2304 "residual_disagree": 0.0504557291666666642305 },2306 {2307 "tau": 1.25,2308 "escalated_frac": 0.16292317708333334,2309 "residual_disagree": 0.0475260416666666642310 },2311 {2312 "tau": 1.25,2313 "escalated_frac": 0.16292317708333334,2314 "residual_disagree": 0.0475260416666666642315 },2316 {2317 "tau": 1.25,2318 "escalated_frac": 0.16292317708333334,2319 "residual_disagree": 0.0475260416666666642320 },2321 {2322 "tau": 1.25,2323 "escalated_frac": 0.16292317708333334,2324 "residual_disagree": 0.0475260416666666642325 },2326 {2327 "tau": 1.25,2328 "escalated_frac": 0.16292317708333334,2329 "residual_disagree": 0.0475260416666666642330 },2331 {2332 "tau": 1.375,2333 "escalated_frac": 0.18880208333333334,2334 "residual_disagree": 0.0403645833333333362335 },2336 {2337 "tau": 1.375,2338 "escalated_frac": 0.18880208333333334,2339 "residual_disagree": 0.0403645833333333362340 },2341 {2342 "tau": 1.5,2343 "escalated_frac": 0.19612630208333334,2344 "residual_disagree": 0.03906252345 },2346 {2347 "tau": 1.5,2348 "escalated_frac": 0.19612630208333334,2349 "residual_disagree": 0.03906252350 },2351 {2352 "tau": 1.5,2353 "escalated_frac": 0.19612630208333334,2354 "residual_disagree": 0.03906252355 },2356 {2357 "tau": 1.5,2358 "escalated_frac": 0.19612630208333334,2359 "residual_disagree": 0.03906252360 },2361 {2362 "tau": 1.625,2363 "escalated_frac": 0.21663411458333334,2364 "residual_disagree": 0.032714843752365 },2366 {2367 "tau": 1.75,2368 "escalated_frac": 0.22412109375,2369 "residual_disagree": 0.0304361979166666682370 },2371 {2372 "tau": 1.75,2373 "escalated_frac": 0.22412109375,2374 "residual_disagree": 0.0304361979166666682375 },2376 {2377 "tau": 1.75,2378 "escalated_frac": 0.22412109375,2379 "residual_disagree": 0.0304361979166666682380 },2381 {2382 "tau": 1.75,2383 "escalated_frac": 0.22412109375,2384 "residual_disagree": 0.0304361979166666682385 },2386 {2387 "tau": 1.875,2388 "escalated_frac": 0.24235026041666666,2389 "residual_disagree": 0.0271809895833333322390 },2391 {2392 "tau": 2.0,2393 "escalated_frac": 0.24886067708333334,2394 "residual_disagree": 0.0260416666666666682395 },2396 {2397 "tau": 2.0,2398 "escalated_frac": 0.24886067708333334,2399 "residual_disagree": 0.0260416666666666682400 },2401 {2402 "tau": 2.0,2403 "escalated_frac": 0.24886067708333334,2404 "residual_disagree": 0.0260416666666666682405 },2406 {2407 "tau": 2.0,2408 "escalated_frac": 0.24886067708333334,2409 "residual_disagree": 0.0260416666666666682410 },2411 {2412 "tau": 2.125,2413 "escalated_frac": 0.2688802083333333,2414 "residual_disagree": 0.021972656252415 },2416 {2417 "tau": 2.125,2418 "escalated_frac": 0.2688802083333333,2419 "residual_disagree": 0.021972656252420 },2421 {2422 "tau": 2.25,2423 "escalated_frac": 0.2766927083333333,2424 "residual_disagree": 0.0201822916666666682425 },2426 {2427 "tau": 2.25,2428 "escalated_frac": 0.2766927083333333,2429 "residual_disagree": 0.0201822916666666682430 },2431 {2432 "tau": 2.25,2433 "escalated_frac": 0.2766927083333333,2434 "residual_disagree": 0.0201822916666666682435 },2436 {2437 "tau": 2.375,2438 "escalated_frac": 0.2939453125,2439 "residual_disagree": 0.0179036458333333322440 },2441 {2442 "tau": 2.5,2443 "escalated_frac": 0.3004557291666667,2444 "residual_disagree": 0.0172526041666666682445 },2446 {2447 "tau": 2.5,2448 "escalated_frac": 0.3004557291666667,2449 "residual_disagree": 0.0172526041666666682450 },2451 {2452 "tau": 2.5,2453 "escalated_frac": 0.3004557291666667,2454 "residual_disagree": 0.0172526041666666682455 },2456 {2457 "tau": 2.5,2458 "escalated_frac": 0.3004557291666667,2459 "residual_disagree": 0.0172526041666666682460 },2461 {2462 "tau": 2.625,2463 "escalated_frac": 0.3196614583333333,2464 "residual_disagree": 0.0148111979166666662465 },2466 {2467 "tau": 2.75,2468 "escalated_frac": 0.3263346354166667,2469 "residual_disagree": 0.0138346354166666662470 },2471 {2472 "tau": 2.75,2473 "escalated_frac": 0.3263346354166667,2474 "residual_disagree": 0.0138346354166666662475 },2476 {2477 "tau": 2.75,2478 "escalated_frac": 0.3263346354166667,2479 "residual_disagree": 0.0138346354166666662480 },2481 {2482 "tau": 2.75,2483 "escalated_frac": 0.3263346354166667,2484 "residual_disagree": 0.0138346354166666662485 },2486 {2487 "tau": 2.875,2488 "escalated_frac": 0.3448893229166667,2489 "residual_disagree": 0.011230468752490 },2491 {2492 "tau": 3.0,2493 "escalated_frac": 0.3512369791666667,2494 "residual_disagree": 0.010253906252495 },2496 {2497 "tau": 3.0,2498 "escalated_frac": 0.3512369791666667,2499 "residual_disagree": 0.010253906252500 },2501 {2502 "tau": 3.0,2503 "escalated_frac": 0.3512369791666667,2504 "residual_disagree": 0.010253906252505 },2506 {2507 "tau": 3.125,2508 "escalated_frac": 0.3658854166666667,2509 "residual_disagree": 0.0091145833333333342510 },2511 {2512 "tau": 3.25,2513 "escalated_frac": 0.37060546875,2514 "residual_disagree": 0.008300781252515 },2516 {2517 "tau": 3.25,2518 "escalated_frac": 0.37060546875,2519 "residual_disagree": 0.008300781252520 },2521 {2522 "tau": 3.25,2523 "escalated_frac": 0.37060546875,2524 "residual_disagree": 0.008300781252525 },2526 {2527 "tau": 3.375,2528 "escalated_frac": 0.38623046875,2529 "residual_disagree": 0.0071614583333333332530 },2531 {2532 "tau": 3.5,2533 "escalated_frac": 0.3914388020833333,2534 "residual_disagree": 0.0066731770833333332535 },2536 {2537 "tau": 3.5,2538 "escalated_frac": 0.3914388020833333,2539 "residual_disagree": 0.0066731770833333332540 },2541 {2542 "tau": 3.5,2543 "escalated_frac": 0.3914388020833333,2544 "residual_disagree": 0.0066731770833333332545 },2546 {2547 "tau": 3.5521356783919487,2548 "escalated_frac": 0.4070638020833333,2549 "residual_disagree": 0.005371093752550 },2551 {2552 "tau": 3.75,2553 "escalated_frac": 0.4112955729166667,2554 "residual_disagree": 0.0050455729166666672555 },2556 {2557 "tau": 3.75,2558 "escalated_frac": 0.4112955729166667,2559 "residual_disagree": 0.0050455729166666672560 },2561 {2562 "tau": 3.75,2563 "escalated_frac": 0.4112955729166667,2564 "residual_disagree": 0.0050455729166666672565 },2566 {2567 "tau": 3.875,2568 "escalated_frac": 0.4231770833333333,2569 "residual_disagree": 0.0042317708333333332570 },2571 {2572 "tau": 4.0,2573 "escalated_frac": 0.42724609375,2574 "residual_disagree": 0.0042317708333333332575 },2576 {2577 "tau": 4.0,2578 "escalated_frac": 0.42724609375,2579 "residual_disagree": 0.0042317708333333332580 },2581 {2582 "tau": 4.125,2583 "escalated_frac": 0.4386393229166667,2584 "residual_disagree": 0.00374348958333333352585 },2586 {2587 "tau": 4.25,2588 "escalated_frac": 0.44287109375,2589 "residual_disagree": 0.00374348958333333352590 },2591 {2592 "tau": 4.25,2593 "escalated_frac": 0.44287109375,2594 "residual_disagree": 0.00374348958333333352595 },2596 {2597 "tau": 4.375,2598 "escalated_frac": 0.4558919270833333,2599 "residual_disagree": 0.00292968752600 },2601 {2602 "tau": 4.5,2603 "escalated_frac": 0.4591471354166667,2604 "residual_disagree": 0.00292968752605 },2606 {2607 "tau": 4.5,2608 "escalated_frac": 0.4591471354166667,2609 "residual_disagree": 0.00292968752610 },2611 {2612 "tau": 4.625,2613 "escalated_frac": 0.4698893229166667,2614 "residual_disagree": 0.002441406252615 },2616 {2617 "tau": 4.75,2618 "escalated_frac": 0.4737955729166667,2619 "residual_disagree": 0.002441406252620 },2621 {2622 "tau": 4.75,2623 "escalated_frac": 0.4737955729166667,2624 "residual_disagree": 0.002441406252625 },2626 {2627 "tau": 4.875,2628 "escalated_frac": 0.4850260416666667,2629 "residual_disagree": 0.0019531252630 },2631 {2632 "tau": 5.0,2633 "escalated_frac": 0.4889322916666667,2634 "residual_disagree": 0.0019531252635 },2636 {2637 "tau": 5.0,2638 "escalated_frac": 0.4889322916666667,2639 "residual_disagree": 0.0019531252640 },2641 {2642 "tau": 5.125,2643 "escalated_frac": 0.49853515625,2644 "residual_disagree": 0.0019531252645 },2646 {2647 "tau": 5.25,2648 "escalated_frac": 0.50390625,2649 "residual_disagree": 0.0019531252650 },2651 {2652 "tau": 5.25,2653 "escalated_frac": 0.50390625,2654 "residual_disagree": 0.0019531252655 },2656 {2657 "tau": 5.5,2658 "escalated_frac": 0.5149739583333334,2659 "residual_disagree": 0.00179036458333333332660 },2661 {2662 "tau": 5.5,2663 "escalated_frac": 0.5149739583333334,2664 "residual_disagree": 0.00179036458333333332665 },2666 {2667 "tau": 5.625,2668 "escalated_frac": 0.5255533854166666,2669 "residual_disagree": 0.00162760416666666672670 },2671 {2672 "tau": 5.75,2673 "escalated_frac": 0.5284830729166666,2674 "residual_disagree": 0.00162760416666666672675 },2676 {2677 "tau": 5.875,2678 "escalated_frac": 0.537109375,2679 "residual_disagree": 0.00130208333333333332680 },2681 {2682 "tau": 6.0,2683 "escalated_frac": 0.5413411458333334,2684 "residual_disagree": 0.00130208333333333332685 },2686 {2687 "tau": 6.125,2688 "escalated_frac": 0.5472005208333334,2689 "residual_disagree": 0.00113932291666666672690 },2691 {2692 "tau": 6.25,2693 "escalated_frac": 0.5491536458333334,2694 "residual_disagree": 0.00113932291666666672695 },2696 {2697 "tau": 6.375,2698 "escalated_frac": 0.55712890625,2699 "residual_disagree": 0.00113932291666666672700 },2701 {2702 "tau": 6.5,2703 "escalated_frac": 0.5597330729166666,2704 "residual_disagree": 0.00113932291666666672705 },2706 {2707 "tau": 6.625,2708 "escalated_frac": 0.5665690104166666,2709 "residual_disagree": 0.00097656252710 },2711 {2712 "tau": 6.75,2713 "escalated_frac": 0.57080078125,2714 "residual_disagree": 0.00097656252715 },2716 {2717 "tau": 6.75,2718 "escalated_frac": 0.57080078125,2719 "residual_disagree": 0.00097656252720 },2721 {2722 "tau": 7.0,2723 "escalated_frac": 0.58154296875,2724 "residual_disagree": 0.000488281252725 },2726 {2727 "tau": 7.0,2728 "escalated_frac": 0.58154296875,2729 "residual_disagree": 0.000488281252730 },2731 {2732 "tau": 7.197864321608051,2733 "escalated_frac": 0.5929361979166666,2734 "residual_disagree": 0.00032552083333333332735 },2736 {2737 "tau": 7.25,2738 "escalated_frac": 0.5929361979166666,2739 "residual_disagree": 0.00032552083333333332740 },2741 {2742 "tau": 7.375,2743 "escalated_frac": 0.6013997395833334,2744 "residual_disagree": 0.00032552083333333332745 },2746 {2747 "tau": 7.5,2748 "escalated_frac": 0.6046549479166666,2749 "residual_disagree": 0.00032552083333333332750 },2751 {2752 "tau": 7.75,2753 "escalated_frac": 0.6123046875,2754 "residual_disagree": 0.00032552083333333332755 },2756 {2757 "tau": 7.875,2758 "escalated_frac": 0.6178385416666666,2759 "residual_disagree": 0.00032552083333333332760 },2761 {2762 "tau": 8.0,2763 "escalated_frac": 0.6214192708333334,2764 "residual_disagree": 0.00032552083333333332765 },2766 {2767 "tau": 8.125,2768 "escalated_frac": 0.6272786458333334,2769 "residual_disagree": 0.00032552083333333332770 },2771 {2772 "tau": 8.25,2773 "escalated_frac": 0.6316731770833334,2774 "residual_disagree": 0.00032552083333333332775 },2776 {2777 "tau": 8.375,2778 "escalated_frac": 0.6376953125,2779 "residual_disagree": 0.00032552083333333332780 },2781 {2782 "tau": 8.5,2783 "escalated_frac": 0.640625,2784 "residual_disagree": 0.00032552083333333332785 },2786 {2787 "tau": 8.625,2788 "escalated_frac": 0.6466471354166666,2789 "residual_disagree": 0.00032552083333333332790 },2791 {2792 "tau": 8.75,2793 "escalated_frac": 0.650390625,2794 "residual_disagree": 0.00032552083333333332795 },2796 {2797 "tau": 8.875,2798 "escalated_frac": 0.6555989583333334,2799 "residual_disagree": 0.00032552083333333332800 },2801 {2802 "tau": 9.0,2803 "escalated_frac": 0.6598307291666666,2804 "residual_disagree": 0.00032552083333333332805 },2806 {2807 "tau": 9.125,2808 "escalated_frac": 0.6653645833333334,2809 "residual_disagree": 0.02810 },2811 {2812 "tau": 9.25,2813 "escalated_frac": 0.6686197916666666,2814 "residual_disagree": 0.02815 },2816 {2817 "tau": 9.5,2818 "escalated_frac": 0.6770833333333334,2819 "residual_disagree": 0.02820 },2821 {2822 "tau": 9.5,2823 "escalated_frac": 0.6770833333333334,2824 "residual_disagree": 0.02825 },2826 {2827 "tau": 9.75,2828 "escalated_frac": 0.6861979166666666,2829 "residual_disagree": 0.02830 },2831 {2832 "tau": 9.9962311557789,2833 "escalated_frac": 0.693359375,2834 "residual_disagree": 0.02835 },2836 {2837 "tau": 10.125,2838 "escalated_frac": 0.6971028645833334,2839 "residual_disagree": 0.02840 },2841 {2842 "tau": 10.25,2843 "escalated_frac": 0.6988932291666666,2844 "residual_disagree": 0.02845 },2846 {2847 "tau": 10.5,2848 "escalated_frac": 0.7068684895833334,2849 "residual_disagree": 0.02850 },2851 {2852 "tau": 10.625,2853 "escalated_frac": 0.7120768229166666,2854 "residual_disagree": 0.02855 },2856 {2857 "tau": 10.75,2858 "escalated_frac": 0.71435546875,2859 "residual_disagree": 0.02860 },2861 {2862 "tau": 11.0,2863 "escalated_frac": 0.7220052083333334,2864 "residual_disagree": 0.02865 },2866 {2867 "tau": 11.125,2868 "escalated_frac": 0.7275390625,2869 "residual_disagree": 0.02870 },2871 {2872 "tau": 11.375,2873 "escalated_frac": 0.7327473958333334,2874 "residual_disagree": 0.02875 },2876 {2877 "tau": 11.5,2878 "escalated_frac": 0.7360026041666666,2879 "residual_disagree": 0.02880 },2881 {2882 "tau": 11.75,2883 "escalated_frac": 0.7410481770833334,2884 "residual_disagree": 0.02885 },2886 {2887 "tau": 12.0,2888 "escalated_frac": 0.7478841145833334,2889 "residual_disagree": 0.02890 },2891 {2892 "tau": 12.25,2893 "escalated_frac": 0.7529296875,2894 "residual_disagree": 0.02895 },2896 {2897 "tau": 12.375,2898 "escalated_frac": 0.7566731770833334,2899 "residual_disagree": 0.02900 },2901 {2902 "tau": 12.75,2903 "escalated_frac": 0.7630208333333334,2904 "residual_disagree": 0.02905 },2906 {2907 "tau": 12.875,2908 "escalated_frac": 0.7667643229166666,2909 "residual_disagree": 0.02910 },2911 {2912 "tau": 13.125,2913 "escalated_frac": 0.7734375,2914 "residual_disagree": 0.02915 },2916 {2917 "tau": 13.343592964824097,2918 "escalated_frac": 0.77880859375,2919 "residual_disagree": 0.02920 },2921 {2922 "tau": 13.5,2923 "escalated_frac": 0.78173828125,2924 "residual_disagree": 0.02925 },2926 {2927 "tau": 13.625,2928 "escalated_frac": 0.7880859375,2929 "residual_disagree": 0.02930 },2931 {2932 "tau": 13.875,2933 "escalated_frac": 0.79345703125,2934 "residual_disagree": 0.02935 },2936 {2937 "tau": 14.0,2938 "escalated_frac": 0.7972005208333334,2939 "residual_disagree": 0.02940 },2941 {2942 "tau": 14.25,2943 "escalated_frac": 0.8025716145833334,2944 "residual_disagree": 0.02945 },2946 {2947 "tau": 14.375,2948 "escalated_frac": 0.8079427083333334,2949 "residual_disagree": 0.02950 },2951 {2952 "tau": 14.5,2953 "escalated_frac": 0.8103841145833334,2954 "residual_disagree": 0.02955 },2956 {2957 "tau": 14.75,2958 "escalated_frac": 0.8170572916666666,2959 "residual_disagree": 0.02960 },2961 {2962 "tau": 14.875,2963 "escalated_frac": 0.8212890625,2964 "residual_disagree": 0.02965 },2966 {2967 "tau": 15.180276381909607,2968 "escalated_frac": 0.8291015625,2969 "residual_disagree": 0.02970 },2971 {2972 "tau": 15.5,2973 "escalated_frac": 0.8331705729166666,2974 "residual_disagree": 0.02975 },2976 {2977 "tau": 15.75,2978 "escalated_frac": 0.8377278645833334,2979 "residual_disagree": 0.02980 },2981 {2982 "tau": 16.0,2983 "escalated_frac": 0.8435872395833334,2984 "residual_disagree": 0.02985 },2986 {2987 "tau": 16.125,2988 "escalated_frac": 0.84814453125,2989 "residual_disagree": 0.02990 },2991 {2992 "tau": 16.375,2993 "escalated_frac": 0.8528645833333334,2994 "residual_disagree": 0.02995 },2996 {2997 "tau": 16.625,2998 "escalated_frac": 0.8583984375,2999 "residual_disagree": 0.03000 },3001 {3002 "tau": 16.815954773869294,3003 "escalated_frac": 0.8642578125,3004 "residual_disagree": 0.03005 },3006 {3007 "tau": 17.125,3008 "escalated_frac": 0.8680013020833334,3009 "residual_disagree": 0.03010 },3011 {3012 "tau": 17.375,3013 "escalated_frac": 0.8738606770833334,3014 "residual_disagree": 0.03015 },3016 {3017 "tau": 17.625,3018 "escalated_frac": 0.8782552083333334,3019 "residual_disagree": 0.03020 },3021 {3022 "tau": 17.75062814070361,3023 "escalated_frac": 0.8844401041666666,3024 "residual_disagree": 0.03025 },3026 {3027 "tau": 18.125,3028 "escalated_frac": 0.8880208333333334,3029 "residual_disagree": 0.03030 },3031 {3032 "tau": 18.5,3033 "escalated_frac": 0.8937174479166666,3034 "residual_disagree": 0.03035 },3036 {3037 "tau": 18.875,3038 "escalated_frac": 0.8990885416666666,3039 "residual_disagree": 0.03040 },3041 {3042 "tau": 19.125,3043 "escalated_frac": 0.9025065104166666,3044 "residual_disagree": 0.03045 },3046 {3047 "tau": 19.29396984924631,3048 "escalated_frac": 0.9095052083333334,3049 "residual_disagree": 0.03050 },3051 {3052 "tau": 19.625,3053 "escalated_frac": 0.9138997395833334,3054 "residual_disagree": 0.03055 },3056 {3057 "tau": 19.875,3058 "escalated_frac": 0.9181315104166666,3059 "residual_disagree": 0.03060 },3061 {3062 "tau": 20.25,3063 "escalated_frac": 0.9235026041666666,3064 "residual_disagree": 0.03065 },3066 {3067 "tau": 20.625,3068 "escalated_frac": 0.9285481770833334,3069 "residual_disagree": 0.03070 },3071 {3072 "tau": 21.0,3073 "escalated_frac": 0.93310546875,3074 "residual_disagree": 0.03075 },3076 {3077 "tau": 21.375,3078 "escalated_frac": 0.9383138020833334,3079 "residual_disagree": 0.03080 },3081 {3082 "tau": 21.875,3083 "escalated_frac": 0.9435221354166666,3084 "residual_disagree": 0.03085 },3086 {3087 "tau": 22.25,3088 "escalated_frac": 0.9495442708333334,3089 "residual_disagree": 0.03090 },3091 {3092 "tau": 22.625,3093 "escalated_frac": 0.9542643229166666,3094 "residual_disagree": 0.03095 },3096 {3097 "tau": 23.130653266331706,3098 "escalated_frac": 0.9597981770833334,3099 "residual_disagree": 0.03100 },3101 {3102 "tau": 23.75,3103 "escalated_frac": 0.9645182291666666,3104 "residual_disagree": 0.03105 },3106 {3107 "tau": 24.375,3108 "escalated_frac": 0.9689127604166666,3109 "residual_disagree": 0.03110 },3111 {3112 "tau": 25.206658291457302,3113 "escalated_frac": 0.9747721354166666,3114 "residual_disagree": 0.03115 },3116 {3117 "tau": 25.875,3118 "escalated_frac": 0.97900390625,3119 "residual_disagree": 0.03120 },3121 {3122 "tau": 26.875,3123 "escalated_frac": 0.9845377604166666,3124 "residual_disagree": 0.03125 },3126 {3127 "tau": 28.125,3128 "escalated_frac": 0.9894205729166666,3129 "residual_disagree": 0.03130 },3131 {3132 "tau": 29.766331658291392,3133 "escalated_frac": 0.9949544270833334,3134 "residual_disagree": 0.03135 },3136 {3137 "tau": 35.25,3138 "escalated_frac": 0.9998372395833334,3139 "residual_disagree": 0.03140 }3141 ],3142 "per_domain": {3143 "code": {3144 "agreement_rate": 0.857421875,3145 "auroc": 0.88440415639529453146 },3147 "math": {3148 "agreement_rate": 0.9365234375,3149 "auroc": 0.92111173498034823150 },3151 "chat": {3152 "agreement_rate": 0.8271484375,3153 "auroc": 0.8800552298241053154 },3155 "french": {3156 "agreement_rate": 0.87109375,3157 "auroc": 0.90246636771300453158 },3159 "english": {3160 "agreement_rate": 0.84765625,3161 "auroc": 0.90314457048328013162 },3163 "reasoning": {3164 "agreement_rate": 0.904296875,3165 "auroc": 0.9040309428306963166 }3167 },3168 "escalation_frac_for_99pct": 0.36588541666666673169 },3170 "8": {3171 "bits": 8,3172 "n_positions": 6144,3173 "agreement_rate": 0.9803059895833334,3174 "mean_kl_ref_q": 0.003948633559048176,3175 "auroc_margin_predicts_disagreement": 0.9745905159697743,3176 "median_margin_agree": 6.25,3177 "median_margin_disagree": 0.25,3178 "escalation_curve": [3179 {3180 "tau": 0.0,3181 "escalated_frac": 0.0,3182 "residual_disagree": 0.0196940104166666683183 },3184 {3185 "tau": 0.0,3186 "escalated_frac": 0.0,3187 "residual_disagree": 0.0196940104166666683188 },3189 {3190 "tau": 0.0,3191 "escalated_frac": 0.0,3192 "residual_disagree": 0.0196940104166666683193 },3194 {3195 "tau": 0.125,3196 "escalated_frac": 0.012858072916666666,3197 "residual_disagree": 0.0149739583333333343198 },3199 {3200 "tau": 0.125,3201 "escalated_frac": 0.012858072916666666,3202 "residual_disagree": 0.0149739583333333343203 },3204 {3205 "tau": 0.25,3206 "escalated_frac": 0.022623697916666668,3207 "residual_disagree": 0.0113932291666666663208 },3209 {3210 "tau": 0.25,3211 "escalated_frac": 0.022623697916666668,3212 "residual_disagree": 0.0113932291666666663213 },3214 {3215 "tau": 0.25,3216 "escalated_frac": 0.022623697916666668,3217 "residual_disagree": 0.0113932291666666663218 },3219 {3220 "tau": 0.25,3221 "escalated_frac": 0.022623697916666668,3222 "residual_disagree": 0.0113932291666666663223 },3224 {3225 "tau": 0.25,3226 "escalated_frac": 0.022623697916666668,3227 "residual_disagree": 0.0113932291666666663228 },3229 {3230 "tau": 0.25,3231 "escalated_frac": 0.022623697916666668,3232 "residual_disagree": 0.0113932291666666663233 },3234 {3235 "tau": 0.375,3236 "escalated_frac": 0.051106770833333336,3237 "residual_disagree": 0.00374348958333333353238 },3239 {3240 "tau": 0.5,3241 "escalated_frac": 0.059895833333333336,3242 "residual_disagree": 0.00276692708333333353243 },3244 {3245 "tau": 0.5,3246 "escalated_frac": 0.059895833333333336,3247 "residual_disagree": 0.00276692708333333353248 },3249 {3250 "tau": 0.5,3251 "escalated_frac": 0.059895833333333336,3252 "residual_disagree": 0.00276692708333333353253 },3254 {3255 "tau": 0.5,3256 "escalated_frac": 0.059895833333333336,3257 "residual_disagree": 0.00276692708333333353258 },3259 {3260 "tau": 0.5,3261 "escalated_frac": 0.059895833333333336,3262 "residual_disagree": 0.00276692708333333353263 },3264 {3265 "tau": 0.625,3266 "escalated_frac": 0.08154296875,3267 "residual_disagree": 0.00113932291666666673268 },3269 {3270 "tau": 0.75,3271 "escalated_frac": 0.08935546875,3272 "residual_disagree": 0.00113932291666666673273 },3274 {3275 "tau": 0.75,3276 "escalated_frac": 0.08935546875,3277 "residual_disagree": 0.00113932291666666673278 },3279 {3280 "tau": 0.75,3281 "escalated_frac": 0.08935546875,3282 "residual_disagree": 0.00113932291666666673283 },3284 {3285 "tau": 0.75,3286 "escalated_frac": 0.08935546875,3287 "residual_disagree": 0.00113932291666666673288 },3289 {3290 "tau": 0.75,3291 "escalated_frac": 0.08935546875,3292 "residual_disagree": 0.00113932291666666673293 },3294 {3295 "tau": 0.875,3296 "escalated_frac": 0.1142578125,3297 "residual_disagree": 0.000488281253298 },3299 {3300 "tau": 0.875,3301 "escalated_frac": 0.1142578125,3302 "residual_disagree": 0.000488281253303 },3304 {3305 "tau": 1.0,3306 "escalated_frac": 0.12109375,3307 "residual_disagree": 0.000488281253308 },3309 {3310 "tau": 1.0,3311 "escalated_frac": 0.12109375,3312 "residual_disagree": 0.000488281253313 },3314 {3315 "tau": 1.0,3316 "escalated_frac": 0.12109375,3317 "residual_disagree": 0.000488281253318 },3319 {3320 "tau": 1.0,3321 "escalated_frac": 0.12109375,3322 "residual_disagree": 0.000488281253323 },3324 {3325 "tau": 1.125,3326 "escalated_frac": 0.14111328125,3327 "residual_disagree": 0.00032552083333333333328 },3329 {3330 "tau": 1.25,3331 "escalated_frac": 0.14713541666666666,3332 "residual_disagree": 0.00032552083333333333333 },3334 {3335 "tau": 1.25,3336 "escalated_frac": 0.14713541666666666,3337 "residual_disagree": 0.00032552083333333333338 },3339 {3340 "tau": 1.25,3341 "escalated_frac": 0.14713541666666666,3342 "residual_disagree": 0.00032552083333333333343 },3344 {3345 "tau": 1.25,3346 "escalated_frac": 0.14713541666666666,3347 "residual_disagree": 0.00032552083333333333348 },3349 {3350 "tau": 1.375,3351 "escalated_frac": 0.17041015625,3352 "residual_disagree": 0.03353 },3354 {3355 "tau": 1.5,3356 "escalated_frac": 0.1748046875,3357 "residual_disagree": 0.03358 },3359 {3360 "tau": 1.5,3361 "escalated_frac": 0.1748046875,3362 "residual_disagree": 0.03363 },3364 {3365 "tau": 1.5,3366 "escalated_frac": 0.1748046875,3367 "residual_disagree": 0.03368 },3369 {3370 "tau": 1.5,3371 "escalated_frac": 0.1748046875,3372 "residual_disagree": 0.03373 },3374 {3375 "tau": 1.625,3376 "escalated_frac": 0.19547526041666666,3377 "residual_disagree": 0.03378 },3379 {3380 "tau": 1.75,3381 "escalated_frac": 0.20084635416666666,3382 "residual_disagree": 0.03383 },3384 {3385 "tau": 1.75,3386 "escalated_frac": 0.20084635416666666,3387 "residual_disagree": 0.03388 },3389 {3390 "tau": 1.75,3391 "escalated_frac": 0.20084635416666666,3392 "residual_disagree": 0.03393 },3394 {3395 "tau": 1.75,3396 "escalated_frac": 0.20084635416666666,3397 "residual_disagree": 0.03398 },3399 {3400 "tau": 1.75,3401 "escalated_frac": 0.20084635416666666,3402 "residual_disagree": 0.03403 },3404 {3405 "tau": 1.875,3406 "escalated_frac": 0.22233072916666666,3407 "residual_disagree": 0.03408 },3409 {3410 "tau": 2.0,3411 "escalated_frac": 0.22672526041666666,3412 "residual_disagree": 0.03413 },3414 {3415 "tau": 2.0,3416 "escalated_frac": 0.22672526041666666,3417 "residual_disagree": 0.03418 },3419 {3420 "tau": 2.0,3421 "escalated_frac": 0.22672526041666666,3422 "residual_disagree": 0.03423 },3424 {3425 "tau": 2.125,3426 "escalated_frac": 0.2431640625,3427 "residual_disagree": 0.03428 },3429 {3430 "tau": 2.25,3431 "escalated_frac": 0.24755859375,3432 "residual_disagree": 0.03433 },3434 {3435 "tau": 2.25,3436 "escalated_frac": 0.24755859375,3437 "residual_disagree": 0.03438 },3439 {3440 "tau": 2.25,3441 "escalated_frac": 0.24755859375,3442 "residual_disagree": 0.03443 },3444 {3445 "tau": 2.375,3446 "escalated_frac": 0.2644856770833333,3447 "residual_disagree": 0.03448 },3449 {3450 "tau": 2.5,3451 "escalated_frac": 0.2696940104166667,3452 "residual_disagree": 0.03453 },3454 {3455 "tau": 2.5,3456 "escalated_frac": 0.2696940104166667,3457 "residual_disagree": 0.03458 },3459 {3460 "tau": 2.5,3461 "escalated_frac": 0.2696940104166667,3462 "residual_disagree": 0.03463 },3464 {3465 "tau": 2.5,3466 "escalated_frac": 0.2696940104166667,3467 "residual_disagree": 0.03468 },3469 {3470 "tau": 2.625,3471 "escalated_frac": 0.2872721354166667,3472 "residual_disagree": 0.03473 },3474 {3475 "tau": 2.75,3476 "escalated_frac": 0.2931315104166667,3477 "residual_disagree": 0.03478 },3479 {3480 "tau": 2.75,3481 "escalated_frac": 0.2931315104166667,3482 "residual_disagree": 0.03483 },3484 {3485 "tau": 2.75,3486 "escalated_frac": 0.2931315104166667,3487 "residual_disagree": 0.03488 },3489 {3490 "tau": 2.875,3491 "escalated_frac": 0.3089192708333333,3492 "residual_disagree": 0.03493 },3494 {3495 "tau": 3.0,3496 "escalated_frac": 0.3136393229166667,3497 "residual_disagree": 0.03498 },3499 {3500 "tau": 3.0,3501 "escalated_frac": 0.3136393229166667,3502 "residual_disagree": 0.03503 },3504 {3505 "tau": 3.0,3506 "escalated_frac": 0.3136393229166667,3507 "residual_disagree": 0.03508 },3509 {3510 "tau": 3.125,3511 "escalated_frac": 0.3304036458333333,3512 "residual_disagree": 0.03513 },3514 {3515 "tau": 3.25,3516 "escalated_frac": 0.33447265625,3517 "residual_disagree": 0.03518 },3519 {3520 "tau": 3.25,3521 "escalated_frac": 0.33447265625,3522 "residual_disagree": 0.03523 },3524 {3525 "tau": 3.25,3526 "escalated_frac": 0.33447265625,3527 "residual_disagree": 0.03528 },3529 {3530 "tau": 3.375,3531 "escalated_frac": 0.3504231770833333,3532 "residual_disagree": 0.03533 },3534 {3535 "tau": 3.5,3536 "escalated_frac": 0.35400390625,3537 "residual_disagree": 0.03538 },3539 {3540 "tau": 3.5,3541 "escalated_frac": 0.35400390625,3542 "residual_disagree": 0.03543 },3544 {3545 "tau": 3.5,3546 "escalated_frac": 0.35400390625,3547 "residual_disagree": 0.03548 },3549 {3550 "tau": 3.75,3551 "escalated_frac": 0.37158203125,3552 "residual_disagree": 0.03553 },3554 {3555 "tau": 3.75,3556 "escalated_frac": 0.37158203125,3557 "residual_disagree": 0.03558 },3559 {3560 "tau": 3.75,3561 "escalated_frac": 0.37158203125,3562 "residual_disagree": 0.03563 },3564 {3565 "tau": 3.75,3566 "escalated_frac": 0.37158203125,3567 "residual_disagree": 0.03568 },3569 {3570 "tau": 3.875,3571 "escalated_frac": 0.3888346354166667,3572 "residual_disagree": 0.03573 },3574 {3575 "tau": 4.0,3576 "escalated_frac": 0.3929036458333333,3577 "residual_disagree": 0.03578 },3579 {3580 "tau": 4.0,3581 "escalated_frac": 0.3929036458333333,3582 "residual_disagree": 0.03583 },3584 {3585 "tau": 4.177135678391949,3586 "escalated_frac": 0.4070638020833333,3587 "residual_disagree": 0.03588 },3589 {3590 "tau": 4.25,3591 "escalated_frac": 0.4070638020833333,3592 "residual_disagree": 0.03593 },3594 {3595 "tau": 4.25,3596 "escalated_frac": 0.4070638020833333,3597 "residual_disagree": 0.03598 },3599 {3600 "tau": 4.375,3601 "escalated_frac": 0.42138671875,3602 "residual_disagree": 0.03603 },3604 {3605 "tau": 4.5,3606 "escalated_frac": 0.42529296875,3607 "residual_disagree": 0.03608 },3609 {3610 "tau": 4.5,3611 "escalated_frac": 0.42529296875,3612 "residual_disagree": 0.03613 },3614 {3615 "tau": 4.625,3616 "escalated_frac": 0.4358723958333333,3617 "residual_disagree": 0.03618 },3619 {3620 "tau": 4.75,3621 "escalated_frac": 0.43896484375,3622 "residual_disagree": 0.03623 },3624 {3625 "tau": 4.75,3626 "escalated_frac": 0.43896484375,3627 "residual_disagree": 0.03628 },3629 {3630 "tau": 4.875,3631 "escalated_frac": 0.4503580729166667,3632 "residual_disagree": 0.03633 },3634 {3635 "tau": 5.0,3636 "escalated_frac": 0.4529622395833333,3637 "residual_disagree": 0.03638 },3639 {3640 "tau": 5.0,3641 "escalated_frac": 0.4529622395833333,3642 "residual_disagree": 0.03643 },3644 {3645 "tau": 5.25,3646 "escalated_frac": 0.4658203125,3647 "residual_disagree": 0.03648 },3649 {3650 "tau": 5.25,3651 "escalated_frac": 0.4658203125,3652 "residual_disagree": 0.03653 },3654 {3655 "tau": 5.375,3656 "escalated_frac": 0.4752604166666667,3657 "residual_disagree": 0.03658 },3659 {3660 "tau": 5.5,3661 "escalated_frac": 0.4775390625,3662 "residual_disagree": 0.03663 },3664 {3665 "tau": 5.625,3666 "escalated_frac": 0.4871419270833333,3667 "residual_disagree": 0.03668 },3669 {3670 "tau": 5.75,3671 "escalated_frac": 0.4900716145833333,3672 "residual_disagree": 0.03673 },3674 {3675 "tau": 5.875,3676 "escalated_frac": 0.4972330729166667,3677 "residual_disagree": 0.03678 },3679 {3680 "tau": 6.0,3681 "escalated_frac": 0.49951171875,3682 "residual_disagree": 0.03683 },3684 {3685 "tau": 6.125,3686 "escalated_frac": 0.5066731770833334,3687 "residual_disagree": 0.03688 },3689 {3690 "tau": 6.25,3691 "escalated_frac": 0.50927734375,3692 "residual_disagree": 0.03693 },3694 {3695 "tau": 6.25,3696 "escalated_frac": 0.50927734375,3697 "residual_disagree": 0.03698 },3699 {3700 "tau": 6.5,3701 "escalated_frac": 0.52099609375,3702 "residual_disagree": 0.03703 },3704 {3705 "tau": 6.5,3706 "escalated_frac": 0.52099609375,3707 "residual_disagree": 0.03708 },3709 {3710 "tau": 6.75,3711 "escalated_frac": 0.53173828125,3712 "residual_disagree": 0.03713 },3714 {3715 "tau": 6.75,3716 "escalated_frac": 0.53173828125,3717 "residual_disagree": 0.03718 },3719 {3720 "tau": 6.875,3721 "escalated_frac": 0.541015625,3722 "residual_disagree": 0.03723 },3724 {3725 "tau": 7.0,3726 "escalated_frac": 0.54296875,3727 "residual_disagree": 0.03728 },3729 {3730 "tau": 7.125,3731 "escalated_frac": 0.5499674479166666,3732 "residual_disagree": 0.03733 },3734 {3735 "tau": 7.25,3736 "escalated_frac": 0.5540364583333334,3737 "residual_disagree": 0.03738 },3739 {3740 "tau": 7.5,3741 "escalated_frac": 0.5626627604166666,3742 "residual_disagree": 0.03743 },3744 {3745 "tau": 7.5,3746 "escalated_frac": 0.5626627604166666,3747 "residual_disagree": 0.03748 },3749 {3750 "tau": 7.75,3751 "escalated_frac": 0.5709635416666666,3752 "residual_disagree": 0.03753 },3754 {3755 "tau": 7.75,3756 "escalated_frac": 0.5709635416666666,3757 "residual_disagree": 0.03758 },3759 {3760 "tau": 8.0,3761 "escalated_frac": 0.5802408854166666,3762 "residual_disagree": 0.03763 },3764 {3765 "tau": 8.125,3766 "escalated_frac": 0.5852864583333334,3767 "residual_disagree": 0.03768 },3769 {3770 "tau": 8.25,3771 "escalated_frac": 0.5880533854166666,3772 "residual_disagree": 0.03773 },3774 {3775 "tau": 8.5,3776 "escalated_frac": 0.59765625,3777 "residual_disagree": 0.03778 },3779 {3780 "tau": 8.5,3781 "escalated_frac": 0.59765625,3782 "residual_disagree": 0.03783 },3784 {3785 "tau": 8.648869346733647,3786 "escalated_frac": 0.6080729166666666,3787 "residual_disagree": 0.03788 },3789 {3790 "tau": 8.75,3791 "escalated_frac": 0.6080729166666666,3792 "residual_disagree": 0.03793 },3794 {3795 "tau": 9.0,3796 "escalated_frac": 0.6160481770833334,3797 "residual_disagree": 0.03798 },3799 {3800 "tau": 9.125,3801 "escalated_frac": 0.6222330729166666,3802 "residual_disagree": 0.03803 },3804 {3805 "tau": 9.25,3806 "escalated_frac": 0.6253255208333334,3807 "residual_disagree": 0.03808 },3809 {3810 "tau": 9.375,3811 "escalated_frac": 0.6326497395833334,3812 "residual_disagree": 0.03813 },3814 {3815 "tau": 9.5,3816 "escalated_frac": 0.63623046875,3817 "residual_disagree": 0.03818 },3819 {3820 "tau": 9.625,3821 "escalated_frac": 0.6416015625,3822 "residual_disagree": 0.03823 },3824 {3825 "tau": 9.75,3826 "escalated_frac": 0.6433919270833334,3827 "residual_disagree": 0.03828 },3829 {3830 "tau": 10.0,3831 "escalated_frac": 0.6531575520833334,3832 "residual_disagree": 0.03833 },3834 {3835 "tau": 10.0,3836 "escalated_frac": 0.6531575520833334,3837 "residual_disagree": 0.03838 },3839 {3840 "tau": 10.25,3841 "escalated_frac": 0.66064453125,3842 "residual_disagree": 0.03843 },3844 {3845 "tau": 10.5,3846 "escalated_frac": 0.6673177083333334,3847 "residual_disagree": 0.03848 },3849 {3850 "tau": 10.75,3851 "escalated_frac": 0.6726888020833334,3852 "residual_disagree": 0.03853 },3854 {3855 "tau": 10.75,3856 "escalated_frac": 0.6726888020833334,3857 "residual_disagree": 0.03858 },3859 {3860 "tau": 11.0,3861 "escalated_frac": 0.681640625,3862 "residual_disagree": 0.03863 },3864 {3865 "tau": 11.25,3866 "escalated_frac": 0.6883138020833334,3867 "residual_disagree": 0.03868 },3869 {3870 "tau": 11.3712311557789,3871 "escalated_frac": 0.693359375,3872 "residual_disagree": 0.03873 },3874 {3875 "tau": 11.5,3876 "escalated_frac": 0.6954752604166666,3877 "residual_disagree": 0.03878 },3879 {3880 "tau": 11.75,3881 "escalated_frac": 0.7021484375,3882 "residual_disagree": 0.03883 },3884 {3885 "tau": 11.875,3886 "escalated_frac": 0.70556640625,3887 "residual_disagree": 0.03888 },3889 {3890 "tau": 12.0,3891 "escalated_frac": 0.708984375,3892 "residual_disagree": 0.03893 },3894 {3895 "tau": 12.25,3896 "escalated_frac": 0.71728515625,3897 "residual_disagree": 0.03898 },3899 {3900 "tau": 12.375,3901 "escalated_frac": 0.7205403645833334,3902 "residual_disagree": 0.03903 },3904 {3905 "tau": 12.625,3906 "escalated_frac": 0.7278645833333334,3907 "residual_disagree": 0.03908 },3909 {3910 "tau": 12.75,3911 "escalated_frac": 0.7301432291666666,3912 "residual_disagree": 0.03913 },3914 {3915 "tau": 13.0,3916 "escalated_frac": 0.7371419270833334,3917 "residual_disagree": 0.03918 },3919 {3920 "tau": 13.25,3921 "escalated_frac": 0.7425130208333334,3922 "residual_disagree": 0.03923 },3924 {3925 "tau": 13.5,3926 "escalated_frac": 0.7478841145833334,3927 "residual_disagree": 0.03928 },3929 {3930 "tau": 13.75,3931 "escalated_frac": 0.7527669270833334,3932 "residual_disagree": 0.03933 },3934 {3935 "tau": 14.0,3936 "escalated_frac": 0.7584635416666666,3937 "residual_disagree": 0.03938 },3939 {3940 "tau": 14.125,3941 "escalated_frac": 0.7633463541666666,3942 "residual_disagree": 0.03943 },3944 {3945 "tau": 14.25,3946 "escalated_frac": 0.76611328125,3947 "residual_disagree": 0.03948 },3949 {3950 "tau": 14.5,3951 "escalated_frac": 0.7716471354166666,3952 "residual_disagree": 0.03953 },3954 {3955 "tau": 14.75,3956 "escalated_frac": 0.7786458333333334,3957 "residual_disagree": 0.03958 },3959 {3960 "tau": 14.875,3961 "escalated_frac": 0.7830403645833334,3962 "residual_disagree": 0.03963 },3964 {3965 "tau": 15.0,3966 "escalated_frac": 0.7867838541666666,3967 "residual_disagree": 0.03968 },3969 {3970 "tau": 15.25,3971 "escalated_frac": 0.7937825520833334,3972 "residual_disagree": 0.03973 },3974 {3975 "tau": 15.375,3976 "escalated_frac": 0.796875,3977 "residual_disagree": 0.03978 },3979 {3980 "tau": 15.5,3981 "escalated_frac": 0.8009440104166666,3982 "residual_disagree": 0.03983 },3984 {3985 "tau": 15.75,3986 "escalated_frac": 0.8082682291666666,3987 "residual_disagree": 0.03988 },3989 {3990 "tau": 15.875,3991 "escalated_frac": 0.8136393229166666,3992 "residual_disagree": 0.03993 },3994 {3995 "tau": 16.087939698492505,3996 "escalated_frac": 0.8190104166666666,3997 "residual_disagree": 0.03998 },3999 {4000 "tau": 16.375,4001 "escalated_frac": 0.8230794270833334,4002 "residual_disagree": 0.04003 },4004 {4005 "tau": 16.625,4006 "escalated_frac": 0.828125,4007 "residual_disagree": 0.04008 },4009 {4010 "tau": 16.75,4011 "escalated_frac": 0.8312174479166666,4012 "residual_disagree": 0.04013 },4014 {4015 "tau": 17.0,4016 "escalated_frac": 0.8380533854166666,4017 "residual_disagree": 0.04018 },4019 {4020 "tau": 17.25,4021 "escalated_frac": 0.8435872395833334,4022 "residual_disagree": 0.04023 },4024 {4025 "tau": 17.5,4026 "escalated_frac": 0.8479817708333334,4027 "residual_disagree": 0.04028 },4029 {4030 "tau": 17.75,4031 "escalated_frac": 0.8528645833333334,4032 "residual_disagree": 0.04033 },4034 {4035 "tau": 18.0,4036 "escalated_frac": 0.8575846354166666,4037 "residual_disagree": 0.04038 },4039 {4040 "tau": 18.190954773869294,4041 "escalated_frac": 0.8642578125,4042 "residual_disagree": 0.04043 },4044 {4045 "tau": 18.5,4046 "escalated_frac": 0.86767578125,4047 "residual_disagree": 0.04048 },4049 {4050 "tau": 18.75,4051 "escalated_frac": 0.8733723958333334,4052 "residual_disagree": 0.04053 },4054 {4055 "tau": 19.125,4056 "escalated_frac": 0.8790690104166666,4057 "residual_disagree": 0.04058 },4059 {4060 "tau": 19.5,4061 "escalated_frac": 0.88427734375,4062 "residual_disagree": 0.04063 },4064 {4065 "tau": 19.75,4066 "escalated_frac": 0.8885091145833334,4067 "residual_disagree": 0.04068 },4069 {4070 "tau": 20.125,4071 "escalated_frac": 0.8935546875,4072 "residual_disagree": 0.04073 },4074 {4075 "tau": 20.5,4076 "escalated_frac": 0.8982747395833334,4077 "residual_disagree": 0.04078 },4079 {4080 "tau": 20.8103015075377,4081 "escalated_frac": 0.9044596354166666,4082 "residual_disagree": 0.04083 },4084 {4085 "tau": 21.125,4086 "escalated_frac": 0.9090169270833334,4087 "residual_disagree": 0.04088 },4089 {4090 "tau": 21.5,4091 "escalated_frac": 0.91357421875,4092 "residual_disagree": 0.04093 },4094 {4095 "tau": 21.875,4096 "escalated_frac": 0.9182942708333334,4097 "residual_disagree": 0.04098 },4099 {4100 "tau": 22.25,4101 "escalated_frac": 0.9225260416666666,4102 "residual_disagree": 0.04103 },4104 {4105 "tau": 22.625,4106 "escalated_frac": 0.9285481770833334,4107 "residual_disagree": 0.04108 },4109 {4110 "tau": 23.0,4111 "escalated_frac": 0.93310546875,4112 "residual_disagree": 0.04113 },4114 {4115 "tau": 23.5,4116 "escalated_frac": 0.9391276041666666,4117 "residual_disagree": 0.04118 },4119 {4120 "tau": 23.875,4121 "escalated_frac": 0.9436848958333334,4122 "residual_disagree": 0.04123 },4124 {4125 "tau": 24.375,4126 "escalated_frac": 0.9495442708333334,4127 "residual_disagree": 0.04128 },4129 {4130 "tau": 24.88599246231155,4131 "escalated_frac": 0.9547526041666666,4132 "residual_disagree": 0.04133 },4134 {4135 "tau": 25.25,4136 "escalated_frac": 0.9578450520833334,4137 "residual_disagree": 0.04138 },4139 {4140 "tau": 25.875,4141 "escalated_frac": 0.9645182291666666,4142 "residual_disagree": 0.04143 },4144 {4145 "tau": 26.695979899497388,4146 "escalated_frac": 0.9697265625,4147 "residual_disagree": 0.04148 },4149 {4150 "tau": 27.25,4151 "escalated_frac": 0.9744466145833334,4152 "residual_disagree": 0.04153 },4154 {4155 "tau": 28.25,4156 "escalated_frac": 0.9794921875,4157 "residual_disagree": 0.04158 },4159 {4160 "tau": 29.5,4161 "escalated_frac": 0.9845377604166666,4162 "residual_disagree": 0.04163 },4164 {4165 "tau": 30.5,4166 "escalated_frac": 0.98974609375,4167 "residual_disagree": 0.04168 },4169 {4170 "tau": 32.032663316582784,4171 "escalated_frac": 0.9949544270833334,4172 "residual_disagree": 0.04173 },4174 {4175 "tau": 37.5,4176 "escalated_frac": 0.9998372395833334,4177 "residual_disagree": 0.04178 }4179 ],4180 "per_domain": {4181 "code": {4182 "agreement_rate": 0.982421875,4183 "auroc": 0.96112215595316994184 },4185 "math": {4186 "agreement_rate": 0.9912109375,4187 "auroc": 0.98713738368910794188 },4189 "chat": {4190 "agreement_rate": 0.974609375,4191 "auroc": 0.9689378757515034192 },4193 "french": {4194 "agreement_rate": 0.978515625,4195 "auroc": 0.9818544728724374196 },4197 "english": {4198 "agreement_rate": 0.9677734375,4199 "auroc": 0.97244901079411674200 },4201 "reasoning": {4202 "agreement_rate": 0.9873046875,4203 "auroc": 0.98006543407136884204 }4205 },4206 "escalation_frac_for_99pct": 0.0511067708333333364207 }4208 }4209}