spb/localvm-research Public License
Running LLMs larger than memory on a consumer Mac — falsification-driven research: margin-gated deferred refinement, out-of-core verification on Apple Silicon. TR-01 published.
Python 63.2%
JavaScript 23.5%
CSS 11.8%
Shell 0.9%
Makefile 0.5%
1{2 "experiment": "expG_decision_stability",3 "author": "Simon-Pierre Boucher",4 "contact": "contact@spboucher.ai",5 "manifest": {6 "author": "Simon-Pierre Boucher",7 "contact": "contact@spboucher.ai",8 "project": "localvm-research",9 "collected_utc": "2026-08-12T04:26:28.220460+00:00",10 "chip": {11 "brand": "Apple M5 Max",12 "arch": "arm64",13 "cores_total": 18,14 "cores_performance": 6,15 "cores_efficiency": 12,16 "gpu_cores": 4017 },18 "memory": {19 "unified_bytes": 51539607552,20 "unified_gb": 48.0,21 "pagesize": 1638422 },23 "ssd": {24 "model": "APPLE SSD AP2048Z",25 "size": "2 TB",26 "smart_status": "Verified"27 },28 "os": {29 "product": "macOS",30 "version": "27.0",31 "build": "26A5388g",32 "kernel": "27.0.0"33 },34 "software": {35 "python": "3.14.4",36 "mlx": "0.32.0",37 "mlx_lm": "0.31.3",38 "torch": null,39 "numpy": "2.5.2"40 },41 "git": {42 "commit": "2b7d23b36999ff09c3e6ac03a730575e71e79650",43 "dirty_tree": false44 },45 "thermal_level_at_collect": null46 },47 "config": {48 "model": "mlx-community/Qwen3-1.7B-bf16",49 "gen_tokens": 48,50 "bits": "4",51 "per_domain": 2,52 "group_size": 6453 },54 "n_trajectories": 12,55 "results_by_bits": {56 "4": {57 "bits": 4,58 "n_positions": 576,59 "agreement_rate": 0.9236111111111112,60 "mean_kl_ref_q": 0.11886843293905258,61 "auroc_margin_predicts_disagreement": 0.9429895762132604,62 "median_margin_agree": 8.8125,63 "median_margin_disagree": 0.875,64 "escalation_curve": [65 {66 "tau": 0.0,67 "escalated_frac": 0.0,68 "residual_disagree": 0.076388888888888969 },70 {71 "tau": 0.11118090452261309,72 "escalated_frac": 0.005208333333333333,73 "residual_disagree": 0.0729166666666666774 },75 {76 "tau": 0.22236180904522618,77 "escalated_frac": 0.010416666666666666,78 "residual_disagree": 0.0694444444444444579 },80 {81 "tau": 0.25,82 "escalated_frac": 0.010416666666666666,83 "residual_disagree": 0.0694444444444444584 },85 {86 "tau": 0.31972361809045236,87 "escalated_frac": 0.020833333333333332,88 "residual_disagree": 0.062589 },90 {91 "tau": 0.375,92 "escalated_frac": 0.020833333333333332,93 "residual_disagree": 0.062594 },95 {96 "tau": 0.5,97 "escalated_frac": 0.027777777777777776,98 "residual_disagree": 0.05902777777777777699 },100 {101 "tau": 0.5,102 "escalated_frac": 0.027777777777777776,103 "residual_disagree": 0.059027777777777776104 },105 {106 "tau": 0.5,107 "escalated_frac": 0.027777777777777776,108 "residual_disagree": 0.059027777777777776109 },110 {111 "tau": 0.5,112 "escalated_frac": 0.027777777777777776,113 "residual_disagree": 0.059027777777777776114 },115 {116 "tau": 0.5,117 "escalated_frac": 0.027777777777777776,118 "residual_disagree": 0.059027777777777776119 },120 {121 "tau": 0.75,122 "escalated_frac": 0.052083333333333336,123 "residual_disagree": 0.04513888888888889124 },125 {126 "tau": 0.75,127 "escalated_frac": 0.052083333333333336,128 "residual_disagree": 0.04513888888888889129 },130 {131 "tau": 0.8203517587939704,132 "escalated_frac": 0.06597222222222222,133 "residual_disagree": 0.03993055555555555134 },135 {136 "tau": 0.875,137 "escalated_frac": 0.06597222222222222,138 "residual_disagree": 0.03993055555555555139 },140 {141 "tau": 0.9177135678391961,142 "escalated_frac": 0.0763888888888889,143 "residual_disagree": 0.034722222222222224144 },145 {146 "tau": 1.0,147 "escalated_frac": 0.0763888888888889,148 "residual_disagree": 0.034722222222222224149 },150 {151 "tau": 1.0150753768844218,152 "escalated_frac": 0.08680555555555555,153 "residual_disagree": 0.027777777777777776154 },155 {156 "tau": 1.125,157 "escalated_frac": 0.08680555555555555,158 "residual_disagree": 0.027777777777777776159 },160 {161 "tau": 1.25,162 "escalated_frac": 0.09375,163 "residual_disagree": 0.026041666666666668164 },165 {166 "tau": 1.25,167 "escalated_frac": 0.09375,168 "residual_disagree": 0.026041666666666668169 },170 {171 "tau": 1.25,172 "escalated_frac": 0.09375,173 "residual_disagree": 0.026041666666666668174 },175 {176 "tau": 1.25,177 "escalated_frac": 0.09375,178 "residual_disagree": 0.026041666666666668179 },180 {181 "tau": 1.375,182 "escalated_frac": 0.11458333333333333,183 "residual_disagree": 0.019097222222222224184 },185 {186 "tau": 1.5,187 "escalated_frac": 0.11979166666666667,188 "residual_disagree": 0.019097222222222224189 },190 {191 "tau": 1.5,192 "escalated_frac": 0.11979166666666667,193 "residual_disagree": 0.019097222222222224194 },195 {196 "tau": 1.5,197 "escalated_frac": 0.11979166666666667,198 "residual_disagree": 0.019097222222222224199 },200 {201 "tau": 1.5,202 "escalated_frac": 0.11979166666666667,203 "residual_disagree": 0.019097222222222224204 },205 {206 "tau": 1.5,207 "escalated_frac": 0.11979166666666667,208 "residual_disagree": 0.019097222222222224209 },210 {211 "tau": 1.625,212 "escalated_frac": 0.1440972222222222,213 "residual_disagree": 0.013888888888888888214 },215 {216 "tau": 1.75,217 "escalated_frac": 0.14930555555555555,218 "residual_disagree": 0.010416666666666666219 },220 {221 "tau": 1.75,222 "escalated_frac": 0.14930555555555555,223 "residual_disagree": 0.010416666666666666224 },225 {226 "tau": 1.75,227 "escalated_frac": 0.14930555555555555,228 "residual_disagree": 0.010416666666666666229 },230 {231 "tau": 1.7939698492462313,232 "escalated_frac": 0.16666666666666666,233 "residual_disagree": 0.010416666666666666234 },235 {236 "tau": 2.0,237 "escalated_frac": 0.1684027777777778,238 "residual_disagree": 0.010416666666666666239 },240 {241 "tau": 2.0,242 "escalated_frac": 0.1684027777777778,243 "residual_disagree": 0.010416666666666666244 },245 {246 "tau": 2.125,247 "escalated_frac": 0.17881944444444445,248 "residual_disagree": 0.010416666666666666249 },250 {251 "tau": 2.25,252 "escalated_frac": 0.1840277777777778,253 "residual_disagree": 0.010416666666666666254 },255 {256 "tau": 2.25,257 "escalated_frac": 0.1840277777777778,258 "residual_disagree": 0.010416666666666666259 },260 {261 "tau": 2.25,262 "escalated_frac": 0.1840277777777778,263 "residual_disagree": 0.010416666666666666264 },265 {266 "tau": 2.3222361809045236,267 "escalated_frac": 0.2013888888888889,268 "residual_disagree": 0.010416666666666666269 },270 {271 "tau": 2.375,272 "escalated_frac": 0.2013888888888889,273 "residual_disagree": 0.010416666666666666274 },275 {276 "tau": 2.5,277 "escalated_frac": 0.20833333333333334,278 "residual_disagree": 0.010416666666666666279 },280 {281 "tau": 2.5,282 "escalated_frac": 0.20833333333333334,283 "residual_disagree": 0.010416666666666666284 },285 {286 "tau": 2.5,287 "escalated_frac": 0.20833333333333334,288 "residual_disagree": 0.010416666666666666289 },290 {291 "tau": 2.5031407035175874,292 "escalated_frac": 0.22743055555555555,293 "residual_disagree": 0.008680555555555556294 },295 {296 "tau": 2.625,297 "escalated_frac": 0.22743055555555555,298 "residual_disagree": 0.008680555555555556299 },300 {301 "tau": 2.75,302 "escalated_frac": 0.2326388888888889,303 "residual_disagree": 0.006944444444444444304 },305 {306 "tau": 2.75,307 "escalated_frac": 0.2326388888888889,308 "residual_disagree": 0.006944444444444444309 },310 {311 "tau": 2.75,312 "escalated_frac": 0.2326388888888889,313 "residual_disagree": 0.006944444444444444314 },315 {316 "tau": 2.75,317 "escalated_frac": 0.2326388888888889,318 "residual_disagree": 0.006944444444444444319 },320 {321 "tau": 2.75,322 "escalated_frac": 0.2326388888888889,323 "residual_disagree": 0.006944444444444444324 },325 {326 "tau": 2.75,327 "escalated_frac": 0.2326388888888889,328 "residual_disagree": 0.006944444444444444329 },330 {331 "tau": 3.0,332 "escalated_frac": 0.265625,333 "residual_disagree": 0.003472222222222222334 },335 {336 "tau": 3.0037688442211063,337 "escalated_frac": 0.2725694444444444,338 "residual_disagree": 0.003472222222222222339 },340 {341 "tau": 3.25,342 "escalated_frac": 0.2743055555555556,343 "residual_disagree": 0.003472222222222222344 },345 {346 "tau": 3.25,347 "escalated_frac": 0.2743055555555556,348 "residual_disagree": 0.003472222222222222349 },350 {351 "tau": 3.375,352 "escalated_frac": 0.2847222222222222,353 "residual_disagree": 0.003472222222222222354 },355 {356 "tau": 3.448492462311556,357 "escalated_frac": 0.2916666666666667,358 "residual_disagree": 0.003472222222222222359 },360 {361 "tau": 3.5,362 "escalated_frac": 0.2916666666666667,363 "residual_disagree": 0.003472222222222222364 },365 {366 "tau": 3.5,367 "escalated_frac": 0.2916666666666667,368 "residual_disagree": 0.003472222222222222369 },370 {371 "tau": 3.75,372 "escalated_frac": 0.3055555555555556,373 "residual_disagree": 0.003472222222222222374 },375 {376 "tau": 3.75,377 "escalated_frac": 0.3055555555555556,378 "residual_disagree": 0.003472222222222222379 },380 {381 "tau": 3.8793969849246217,382 "escalated_frac": 0.3177083333333333,383 "residual_disagree": 0.003472222222222222384 },385 {386 "tau": 4.0,387 "escalated_frac": 0.3177083333333333,388 "residual_disagree": 0.003472222222222222389 },390 {391 "tau": 4.0,392 "escalated_frac": 0.3177083333333333,393 "residual_disagree": 0.003472222222222222394 },395 {396 "tau": 4.0,397 "escalated_frac": 0.3177083333333333,398 "residual_disagree": 0.003472222222222222399 },400 {401 "tau": 4.0,402 "escalated_frac": 0.3177083333333333,403 "residual_disagree": 0.003472222222222222404 },405 {406 "tau": 4.0,407 "escalated_frac": 0.3177083333333333,408 "residual_disagree": 0.003472222222222222409 },410 {411 "tau": 4.125,412 "escalated_frac": 0.34375,413 "residual_disagree": 0.001736111111111111414 },415 {416 "tau": 4.282663316582916,417 "escalated_frac": 0.3524305555555556,418 "residual_disagree": 0.001736111111111111419 },420 {421 "tau": 4.5,422 "escalated_frac": 0.3541666666666667,423 "residual_disagree": 0.001736111111111111424 },425 {426 "tau": 4.75,427 "escalated_frac": 0.3611111111111111,428 "residual_disagree": 0.0429 },430 {431 "tau": 4.75,432 "escalated_frac": 0.3611111111111111,433 "residual_disagree": 0.0434 },435 {436 "tau": 4.75,437 "escalated_frac": 0.3611111111111111,438 "residual_disagree": 0.0439 },440 {441 "tau": 4.963567839195981,442 "escalated_frac": 0.3767361111111111,443 "residual_disagree": 0.0444 },445 {446 "tau": 5.125,447 "escalated_frac": 0.3802083333333333,448 "residual_disagree": 0.0449 },450 {451 "tau": 5.25,452 "escalated_frac": 0.3836805555555556,453 "residual_disagree": 0.0454 },455 {456 "tau": 5.25,457 "escalated_frac": 0.3836805555555556,458 "residual_disagree": 0.0459 },460 {461 "tau": 5.408291457286431,462 "escalated_frac": 0.3975694444444444,463 "residual_disagree": 0.0464 },465 {466 "tau": 5.5,467 "escalated_frac": 0.3975694444444444,468 "residual_disagree": 0.0469 },470 {471 "tau": 5.505653266331656,472 "escalated_frac": 0.4079861111111111,473 "residual_disagree": 0.0474 },475 {476 "tau": 5.625,477 "escalated_frac": 0.4079861111111111,478 "residual_disagree": 0.0479 },480 {481 "tau": 5.75,482 "escalated_frac": 0.4131944444444444,483 "residual_disagree": 0.0484 },485 {486 "tau": 5.839195979899497,487 "escalated_frac": 0.421875,488 "residual_disagree": 0.0489 },490 {491 "tau": 5.950376884422113,492 "escalated_frac": 0.4270833333333333,493 "residual_disagree": 0.0494 },495 {496 "tau": 6.0,497 "escalated_frac": 0.4270833333333333,498 "residual_disagree": 0.0499 },500 {501 "tau": 6.172738693467338,502 "escalated_frac": 0.4375,503 "residual_disagree": 0.0504 },505 {506 "tau": 6.25,507 "escalated_frac": 0.4375,508 "residual_disagree": 0.0509 },510 {511 "tau": 6.5,512 "escalated_frac": 0.4461805555555556,513 "residual_disagree": 0.0514 },515 {516 "tau": 6.5,517 "escalated_frac": 0.4461805555555556,518 "residual_disagree": 0.0519 },520 {521 "tau": 6.5,522 "escalated_frac": 0.4461805555555556,523 "residual_disagree": 0.0524 },525 {526 "tau": 6.75,527 "escalated_frac": 0.4600694444444444,528 "residual_disagree": 0.0529 },530 {531 "tau": 7.089824120603019,532 "escalated_frac": 0.4670138888888889,533 "residual_disagree": 0.0534 },535 {536 "tau": 7.25,537 "escalated_frac": 0.4704861111111111,538 "residual_disagree": 0.0539 },540 {541 "tau": 7.312185929648244,542 "escalated_frac": 0.4774305555555556,543 "residual_disagree": 0.0544 },545 {546 "tau": 7.423366834170849,547 "escalated_frac": 0.4826388888888889,548 "residual_disagree": 0.0549 },550 {551 "tau": 7.5,552 "escalated_frac": 0.4826388888888889,553 "residual_disagree": 0.0554 },555 {556 "tau": 7.75,557 "escalated_frac": 0.4895833333333333,558 "residual_disagree": 0.0559 },560 {561 "tau": 7.756909547738694,562 "escalated_frac": 0.4982638888888889,563 "residual_disagree": 0.0564 },565 {566 "tau": 7.875,567 "escalated_frac": 0.4982638888888889,568 "residual_disagree": 0.0569 },570 {571 "tau": 8.104271356783919,572 "escalated_frac": 0.5069444444444444,573 "residual_disagree": 0.0574 },575 {576 "tau": 8.125,577 "escalated_frac": 0.5069444444444444,578 "residual_disagree": 0.0579 },580 {581 "tau": 8.32663316582915,582 "escalated_frac": 0.5173611111111112,583 "residual_disagree": 0.0584 },585 {586 "tau": 8.375,587 "escalated_frac": 0.5173611111111112,588 "residual_disagree": 0.0589 },590 {591 "tau": 8.5,592 "escalated_frac": 0.5260416666666666,593 "residual_disagree": 0.0594 },595 {596 "tau": 8.75,597 "escalated_frac": 0.53125,598 "residual_disagree": 0.0599 },600 {601 "tau": 8.7713567839196,602 "escalated_frac": 0.5381944444444444,603 "residual_disagree": 0.0604 },605 {606 "tau": 9.0,607 "escalated_frac": 0.5399305555555556,608 "residual_disagree": 0.0609 },610 {611 "tau": 9.0,612 "escalated_frac": 0.5399305555555556,613 "residual_disagree": 0.0614 },615 {616 "tau": 9.125,617 "escalated_frac": 0.5503472222222222,618 "residual_disagree": 0.0619 },620 {621 "tau": 9.34108040201005,622 "escalated_frac": 0.5572916666666666,623 "residual_disagree": 0.0624 },625 {626 "tau": 9.5,627 "escalated_frac": 0.5590277777777778,628 "residual_disagree": 0.0629 },630 {631 "tau": 9.5,632 "escalated_frac": 0.5590277777777778,633 "residual_disagree": 0.0634 },635 {636 "tau": 9.625,637 "escalated_frac": 0.5694444444444444,638 "residual_disagree": 0.0639 },640 {641 "tau": 9.75,642 "escalated_frac": 0.5763888888888888,643 "residual_disagree": 0.0644 },645 {646 "tau": 9.896984924623112,647 "escalated_frac": 0.5833333333333334,648 "residual_disagree": 0.0649 },650 {651 "tau": 10.125,652 "escalated_frac": 0.5850694444444444,653 "residual_disagree": 0.0654 },655 {656 "tau": 10.25,657 "escalated_frac": 0.5902777777777778,658 "residual_disagree": 0.0659 },660 {661 "tau": 10.375,662 "escalated_frac": 0.59375,663 "residual_disagree": 0.0664 },665 {666 "tau": 10.5,667 "escalated_frac": 0.5989583333333334,668 "residual_disagree": 0.0669 },670 {671 "tau": 10.577889447236181,672 "escalated_frac": 0.6076388888888888,673 "residual_disagree": 0.0674 },675 {676 "tau": 10.625,677 "escalated_frac": 0.6076388888888888,678 "residual_disagree": 0.0679 },680 {681 "tau": 10.75,682 "escalated_frac": 0.6163194444444444,683 "residual_disagree": 0.0684 },685 {686 "tau": 10.875,687 "escalated_frac": 0.6197916666666666,688 "residual_disagree": 0.0689 },690 {691 "tau": 11.0,692 "escalated_frac": 0.6267361111111112,693 "residual_disagree": 0.0694 },695 {696 "tau": 11.125,697 "escalated_frac": 0.6319444444444444,698 "residual_disagree": 0.0699 },700 {701 "tau": 11.244974874371856,702 "escalated_frac": 0.6371527777777778,703 "residual_disagree": 0.0704 },705 {706 "tau": 11.481155778894475,707 "escalated_frac": 0.6423611111111112,708 "residual_disagree": 0.0709 },710 {711 "tau": 11.684673366834176,712 "escalated_frac": 0.6475694444444444,713 "residual_disagree": 0.0714 },715 {716 "tau": 11.75,717 "escalated_frac": 0.6475694444444444,718 "residual_disagree": 0.0719 },720 {721 "tau": 12.0,722 "escalated_frac": 0.65625,723 "residual_disagree": 0.0724 },725 {726 "tau": 12.25,727 "escalated_frac": 0.6614583333333334,728 "residual_disagree": 0.0729 },730 {731 "tau": 12.5,732 "escalated_frac": 0.6666666666666666,733 "residual_disagree": 0.0734 },735 {736 "tau": 12.7964824120603,737 "escalated_frac": 0.6736111111111112,738 "residual_disagree": 0.0739 },740 {741 "tau": 13.00942211055277,742 "escalated_frac": 0.6788194444444444,743 "residual_disagree": 0.0744 },745 {746 "tau": 13.370603015075375,747 "escalated_frac": 0.6822916666666666,748 "residual_disagree": 0.0749 },750 {751 "tau": 13.5,752 "escalated_frac": 0.6857638888888888,753 "residual_disagree": 0.0754 },755 {756 "tau": 13.625,757 "escalated_frac": 0.6892361111111112,758 "residual_disagree": 0.0759 },760 {761 "tau": 13.75,762 "escalated_frac": 0.6944444444444444,763 "residual_disagree": 0.0764 },765 {766 "tau": 13.875,767 "escalated_frac": 0.6996527777777778,768 "residual_disagree": 0.0769 },770 {771 "tau": 13.875,772 "escalated_frac": 0.6996527777777778,773 "residual_disagree": 0.0774 },775 {776 "tau": 13.912688442211056,777 "escalated_frac": 0.7135416666666666,778 "residual_disagree": 0.0779 },780 {781 "tau": 14.148869346733669,782 "escalated_frac": 0.71875,783 "residual_disagree": 0.0784 },785 {786 "tau": 14.375,787 "escalated_frac": 0.7222222222222222,788 "residual_disagree": 0.0789 },790 {791 "tau": 14.375,792 "escalated_frac": 0.7222222222222222,793 "residual_disagree": 0.0794 },795 {796 "tau": 14.375,797 "escalated_frac": 0.7222222222222222,798 "residual_disagree": 0.0799 },800 {801 "tau": 14.5,802 "escalated_frac": 0.734375,803 "residual_disagree": 0.0804 },805 {806 "tau": 14.75,807 "escalated_frac": 0.7413194444444444,808 "residual_disagree": 0.0809 },810 {811 "tau": 14.875,812 "escalated_frac": 0.7447916666666666,813 "residual_disagree": 0.0814 },815 {816 "tau": 15.0,817 "escalated_frac": 0.7517361111111112,818 "residual_disagree": 0.0819 },820 {821 "tau": 15.144158291457288,822 "escalated_frac": 0.7586805555555556,823 "residual_disagree": 0.0824 },825 {826 "tau": 15.375,827 "escalated_frac": 0.7621527777777778,828 "residual_disagree": 0.0829 },830 {831 "tau": 15.3963567839196,832 "escalated_frac": 0.7690972222222222,833 "residual_disagree": 0.0834 },835 {836 "tau": 15.746859296482413,837 "escalated_frac": 0.7725694444444444,838 "residual_disagree": 0.0839 },840 {841 "tau": 15.983040201005025,842 "escalated_frac": 0.7777777777777778,843 "residual_disagree": 0.0844 },845 {846 "tau": 16.0,847 "escalated_frac": 0.7777777777777778,848 "residual_disagree": 0.0849 },850 {851 "tau": 16.08040201005025,852 "escalated_frac": 0.7881944444444444,853 "residual_disagree": 0.0854 },855 {856 "tau": 16.25,857 "escalated_frac": 0.7916666666666666,858 "residual_disagree": 0.0859 },860 {861 "tau": 16.427763819095482,862 "escalated_frac": 0.7986111111111112,863 "residual_disagree": 0.0864 },865 {866 "tau": 16.625,867 "escalated_frac": 0.8003472222222222,868 "residual_disagree": 0.0869 },870 {871 "tau": 16.75,872 "escalated_frac": 0.8055555555555556,873 "residual_disagree": 0.0874 },875 {876 "tau": 16.886306532663312,877 "escalated_frac": 0.8142361111111112,878 "residual_disagree": 0.0879 },880 {881 "tau": 17.0,882 "escalated_frac": 0.8142361111111112,883 "residual_disagree": 0.0884 },885 {886 "tau": 17.125,887 "escalated_frac": 0.8211805555555556,888 "residual_disagree": 0.0889 },890 {891 "tau": 17.219849246231156,892 "escalated_frac": 0.828125,893 "residual_disagree": 0.0894 },895 {896 "tau": 17.5,897 "escalated_frac": 0.8298611111111112,898 "residual_disagree": 0.0899 },900 {901 "tau": 17.625,902 "escalated_frac": 0.8368055555555556,903 "residual_disagree": 0.0904 },905 {906 "tau": 17.625,907 "escalated_frac": 0.8368055555555556,908 "residual_disagree": 0.0909 },910 {911 "tau": 17.875,912 "escalated_frac": 0.8472222222222222,913 "residual_disagree": 0.0914 },915 {916 "tau": 18.05150753768845,917 "escalated_frac": 0.8541666666666666,918 "residual_disagree": 0.0919 },920 {921 "tau": 18.375,922 "escalated_frac": 0.8559027777777778,923 "residual_disagree": 0.0924 },925 {926 "tau": 18.5,927 "escalated_frac": 0.8611111111111112,928 "residual_disagree": 0.0929 },930 {931 "tau": 18.75,932 "escalated_frac": 0.8663194444444444,933 "residual_disagree": 0.0934 },935 {936 "tau": 18.970477386934675,937 "escalated_frac": 0.8732638888888888,938 "residual_disagree": 0.0939 },940 {941 "tau": 19.0,942 "escalated_frac": 0.8732638888888888,943 "residual_disagree": 0.0944 },945 {946 "tau": 19.125,947 "escalated_frac": 0.8802083333333334,948 "residual_disagree": 0.0949 },950 {951 "tau": 19.179020100502512,952 "escalated_frac": 0.8888888888888888,953 "residual_disagree": 0.0954 },955 {956 "tau": 19.375,957 "escalated_frac": 0.8923611111111112,958 "residual_disagree": 0.0959 },960 {961 "tau": 19.526381909547737,962 "escalated_frac": 0.8993055555555556,963 "residual_disagree": 0.0964 },965 {966 "tau": 19.875,967 "escalated_frac": 0.9027777777777778,968 "residual_disagree": 0.0969 },970 {971 "tau": 20.123743718592962,972 "escalated_frac": 0.9079861111111112,973 "residual_disagree": 0.0974 },975 {976 "tau": 20.234924623115575,977 "escalated_frac": 0.9131944444444444,978 "residual_disagree": 0.0979 },980 {981 "tau": 20.25,982 "escalated_frac": 0.9131944444444444,983 "residual_disagree": 0.0984 },985 {986 "tau": 20.5822864321608,987 "escalated_frac": 0.9236111111111112,988 "residual_disagree": 0.0989 },990 {991 "tau": 20.693467336683426,992 "escalated_frac": 0.9288194444444444,993 "residual_disagree": 0.0994 },995 {996 "tau": 20.80464824120604,997 "escalated_frac": 0.9340277777777778,998 "residual_disagree": 0.0999 },1000 {1001 "tau": 21.0,1002 "escalated_frac": 0.9375,1003 "residual_disagree": 0.01004 },1005 {1006 "tau": 21.081030150753747,1007 "escalated_frac": 0.9444444444444444,1008 "residual_disagree": 0.01009 },1010 {1011 "tau": 21.625,1012 "escalated_frac": 0.9461805555555556,1013 "residual_disagree": 0.01014 },1015 {1016 "tau": 21.75,1017 "escalated_frac": 0.9513888888888888,1018 "residual_disagree": 0.01019 },1020 {1021 "tau": 22.1105527638191,1022 "escalated_frac": 0.9583333333333334,1023 "residual_disagree": 0.01024 },1025 {1026 "tau": 22.25,1027 "escalated_frac": 0.9618055555555556,1028 "residual_disagree": 0.01029 },1030 {1031 "tau": 23.123743718592976,1032 "escalated_frac": 0.96875,1033 "residual_disagree": 0.01034 },1035 {1036 "tau": 23.638190954773876,1037 "escalated_frac": 0.9739583333333334,1038 "residual_disagree": 0.01039 },1040 {1041 "tau": 24.858040201005053,1042 "escalated_frac": 0.9791666666666666,1043 "residual_disagree": 0.01044 },1045 {1046 "tau": 27.289572864321627,1047 "escalated_frac": 0.984375,1048 "residual_disagree": 0.01049 },1050 {1051 "tau": 29.402638190954775,1052 "escalated_frac": 0.9895833333333334,1053 "residual_disagree": 0.01054 },1055 {1056 "tau": 31.5,1057 "escalated_frac": 0.9930555555555556,1058 "residual_disagree": 0.01059 },1060 {1061 "tau": 31.75,1062 "escalated_frac": 0.9982638888888888,1063 "residual_disagree": 0.01064 }1065 ],1066 "per_domain": {1067 "code": {1068 "agreement_rate": 0.90625,1069 "auroc": 0.90868454661558111070 },1071 "math": {1072 "agreement_rate": 0.9895833333333334,1073 "auroc": 0.88421052631578941074 },1075 "chat": {1076 "agreement_rate": 0.9375,1077 "auroc": 0.94629629629629631078 },1079 "french": {1080 "agreement_rate": 0.8854166666666666,1081 "auroc": 0.93689839572192511082 },1083 "english": {1084 "agreement_rate": 0.8958333333333334,1085 "auroc": 0.9860465116279071086 },1087 "reasoning": {1088 "agreement_rate": 0.9270833333333334,1089 "auroc": 0.96308186195826651090 }1091 },1092 "escalation_frac_for_99pct": 0.227430555555555551093 }1094 }1095}