{ "experiment": "expG_decision_stability", "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai", "manifest": { "author": "Simon-Pierre Boucher", "contact": "contact@spboucher.ai", "project": "localvm-research", "collected_utc": "2026-08-12T04:27:44.503861+00:00", "chip": { "brand": "Apple M5 Max", "arch": "arm64", "cores_total": 18, "cores_performance": 6, "cores_efficiency": 12, "gpu_cores": 40 }, "memory": { "unified_bytes": 51539607552, "unified_gb": 48.0, "pagesize": 16384 }, "ssd": { "model": "APPLE SSD AP2048Z", "size": "2 TB", "smart_status": "Verified" }, "os": { "product": "macOS", "version": "27.0", "build": "26A5388g", "kernel": "27.0.0" }, "software": { "python": "3.14.4", "mlx": "0.32.0", "mlx_lm": "0.31.3", "torch": null, "numpy": "2.5.2" }, "git": { "commit": "2b7d23b36999ff09c3e6ac03a730575e71e79650", "dirty_tree": true }, "thermal_level_at_collect": null }, "config": { "model": "mlx-community/Qwen3-1.7B-bf16", "gen_tokens": 128, "bits": "2,3,4,8", "per_domain": 8, "group_size": 64 }, "n_trajectories": 48, "results_by_bits": { "2": { "bits": 2, "n_positions": 6144, "agreement_rate": 0.029622395833333332, "mean_kl_ref_q": 11.892386436462402, "auroc_margin_predicts_disagreement": 0.6048301329666643, "median_margin_agree": 1.5, "median_margin_disagree": 1.0, "escalation_curve": [ { "tau": 0.0, "escalated_frac": 0.0, "residual_disagree": 0.9703776041666666 }, { "tau": 0.0, "escalated_frac": 0.0, "residual_disagree": 0.9703776041666666 }, { "tau": 0.0, "escalated_frac": 0.0, "residual_disagree": 0.9703776041666666 }, { "tau": 0.0, "escalated_frac": 0.0, "residual_disagree": 0.9703776041666666 }, { "tau": 0.0, "escalated_frac": 0.0, "residual_disagree": 0.9703776041666666 }, { "tau": 0.0, "escalated_frac": 0.0, "residual_disagree": 0.9703776041666666 }, { "tau": 0.0, "escalated_frac": 0.0, "residual_disagree": 0.9703776041666666 }, { "tau": 0.0, "escalated_frac": 0.0, "residual_disagree": 0.9703776041666666 }, { "tau": 0.0625, "escalated_frac": 0.038411458333333336, "residual_disagree": 0.9327799479166666 }, { "tau": 0.125, "escalated_frac": 0.043619791666666664, "residual_disagree": 0.927734375 }, { "tau": 0.125, "escalated_frac": 0.043619791666666664, "residual_disagree": 0.927734375 }, { "tau": 0.125, "escalated_frac": 0.043619791666666664, "residual_disagree": 0.927734375 }, { "tau": 0.125, "escalated_frac": 0.043619791666666664, "residual_disagree": 0.927734375 }, { "tau": 0.125, "escalated_frac": 0.043619791666666664, "residual_disagree": 0.927734375 }, { "tau": 0.125, "escalated_frac": 0.043619791666666664, "residual_disagree": 0.927734375 }, { "tau": 0.125, "escalated_frac": 0.043619791666666664, "residual_disagree": 0.927734375 }, { "tau": 0.125, "escalated_frac": 0.043619791666666664, "residual_disagree": 0.927734375 }, { "tau": 0.125, "escalated_frac": 0.043619791666666664, "residual_disagree": 0.927734375 }, { "tau": 0.125, "escalated_frac": 0.043619791666666664, "residual_disagree": 0.927734375 }, { "tau": 0.125, "escalated_frac": 0.043619791666666664, "residual_disagree": 0.927734375 }, { "tau": 0.125, "escalated_frac": 0.043619791666666664, "residual_disagree": 0.927734375 }, { "tau": 0.125, "escalated_frac": 0.043619791666666664, "residual_disagree": 0.927734375 }, { "tau": 0.125, "escalated_frac": 0.043619791666666664, "residual_disagree": 0.927734375 }, { "tau": 0.1875, "escalated_frac": 0.11328125, "residual_disagree": 0.8601888020833334 }, { "tau": 0.25, "escalated_frac": 0.119140625, "residual_disagree": 0.8543294270833334 }, { "tau": 0.25, "escalated_frac": 0.119140625, "residual_disagree": 0.8543294270833334 }, { "tau": 0.25, "escalated_frac": 0.119140625, "residual_disagree": 0.8543294270833334 }, { "tau": 0.25, "escalated_frac": 0.119140625, "residual_disagree": 0.8543294270833334 }, { "tau": 0.25, "escalated_frac": 0.119140625, "residual_disagree": 0.8543294270833334 }, { "tau": 0.25, "escalated_frac": 0.119140625, "residual_disagree": 0.8543294270833334 }, { "tau": 0.25, "escalated_frac": 0.119140625, "residual_disagree": 0.8543294270833334 }, { "tau": 0.25, "escalated_frac": 0.119140625, "residual_disagree": 0.8543294270833334 }, { "tau": 0.25, "escalated_frac": 0.119140625, "residual_disagree": 0.8543294270833334 }, { "tau": 0.25, "escalated_frac": 0.119140625, "residual_disagree": 0.8543294270833334 }, { "tau": 0.25, "escalated_frac": 0.119140625, "residual_disagree": 0.8543294270833334 }, { "tau": 0.25, "escalated_frac": 0.119140625, "residual_disagree": 0.8543294270833334 }, { "tau": 0.25, "escalated_frac": 0.119140625, "residual_disagree": 0.8543294270833334 }, { "tau": 0.25, "escalated_frac": 0.119140625, "residual_disagree": 0.8543294270833334 }, { "tau": 0.3125, "escalated_frac": 0.18912760416666666, "residual_disagree": 0.7853190104166666 }, { "tau": 0.375, "escalated_frac": 0.19352213541666666, "residual_disagree": 0.7809244791666666 }, { "tau": 0.375, "escalated_frac": 0.19352213541666666, "residual_disagree": 0.7809244791666666 }, { "tau": 0.375, "escalated_frac": 0.19352213541666666, "residual_disagree": 0.7809244791666666 }, { "tau": 0.375, "escalated_frac": 0.19352213541666666, "residual_disagree": 0.7809244791666666 }, { "tau": 0.375, "escalated_frac": 0.19352213541666666, "residual_disagree": 0.7809244791666666 }, { "tau": 0.375, "escalated_frac": 0.19352213541666666, "residual_disagree": 0.7809244791666666 }, { "tau": 0.375, "escalated_frac": 0.19352213541666666, "residual_disagree": 0.7809244791666666 }, { "tau": 0.375, "escalated_frac": 0.19352213541666666, "residual_disagree": 0.7809244791666666 }, { "tau": 0.375, "escalated_frac": 0.19352213541666666, "residual_disagree": 0.7809244791666666 }, { "tau": 0.375, "escalated_frac": 0.19352213541666666, "residual_disagree": 0.7809244791666666 }, { "tau": 0.375, "escalated_frac": 0.19352213541666666, "residual_disagree": 0.7809244791666666 }, { "tau": 0.375, "escalated_frac": 0.19352213541666666, "residual_disagree": 0.7809244791666666 }, { "tau": 0.375, "escalated_frac": 0.19352213541666666, "residual_disagree": 0.7809244791666666 }, { "tau": 0.4375, "escalated_frac": 0.25830078125, "residual_disagree": 0.7176106770833334 }, { "tau": 0.5, "escalated_frac": 0.2630208333333333, "residual_disagree": 0.712890625 }, { "tau": 0.5, "escalated_frac": 0.2630208333333333, "residual_disagree": 0.712890625 }, { "tau": 0.5, "escalated_frac": 0.2630208333333333, "residual_disagree": 0.712890625 }, { "tau": 0.5, "escalated_frac": 0.2630208333333333, "residual_disagree": 0.712890625 }, { "tau": 0.5, "escalated_frac": 0.2630208333333333, "residual_disagree": 0.712890625 }, { "tau": 0.5, "escalated_frac": 0.2630208333333333, "residual_disagree": 0.712890625 }, { "tau": 0.5, "escalated_frac": 0.2630208333333333, "residual_disagree": 0.712890625 }, { "tau": 0.5, "escalated_frac": 0.2630208333333333, "residual_disagree": 0.712890625 }, { "tau": 0.5, "escalated_frac": 0.2630208333333333, "residual_disagree": 0.712890625 }, { "tau": 0.5, "escalated_frac": 0.2630208333333333, "residual_disagree": 0.712890625 }, { "tau": 0.5, "escalated_frac": 0.2630208333333333, "residual_disagree": 0.712890625 }, { "tau": 0.5, "escalated_frac": 0.2630208333333333, "residual_disagree": 0.712890625 }, { "tau": 0.5625, "escalated_frac": 0.3224283854166667, "residual_disagree": 0.65478515625 }, { "tau": 0.625, "escalated_frac": 0.328125, "residual_disagree": 0.6490885416666666 }, { "tau": 0.625, "escalated_frac": 0.328125, "residual_disagree": 0.6490885416666666 }, { "tau": 0.625, "escalated_frac": 0.328125, "residual_disagree": 0.6490885416666666 }, { "tau": 0.625, "escalated_frac": 0.328125, "residual_disagree": 0.6490885416666666 }, { "tau": 0.625, "escalated_frac": 0.328125, "residual_disagree": 0.6490885416666666 }, { "tau": 0.625, "escalated_frac": 0.328125, "residual_disagree": 0.6490885416666666 }, { "tau": 0.625, "escalated_frac": 0.328125, "residual_disagree": 0.6490885416666666 }, { "tau": 0.625, "escalated_frac": 0.328125, "residual_disagree": 0.6490885416666666 }, { "tau": 0.625, "escalated_frac": 0.328125, "residual_disagree": 0.6490885416666666 }, { "tau": 0.625, "escalated_frac": 0.328125, "residual_disagree": 0.6490885416666666 }, { "tau": 0.625, "escalated_frac": 0.328125, "residual_disagree": 0.6490885416666666 }, { "tau": 0.75, "escalated_frac": 0.3841145833333333, "residual_disagree": 0.5944010416666666 }, { "tau": 0.75, "escalated_frac": 0.3841145833333333, "residual_disagree": 0.5944010416666666 }, { "tau": 0.75, "escalated_frac": 0.3841145833333333, "residual_disagree": 0.5944010416666666 }, { "tau": 0.75, "escalated_frac": 0.3841145833333333, "residual_disagree": 0.5944010416666666 }, { "tau": 0.75, "escalated_frac": 0.3841145833333333, "residual_disagree": 0.5944010416666666 }, { "tau": 0.75, "escalated_frac": 0.3841145833333333, "residual_disagree": 0.5944010416666666 }, { "tau": 0.75, "escalated_frac": 0.3841145833333333, "residual_disagree": 0.5944010416666666 }, { "tau": 0.75, "escalated_frac": 0.3841145833333333, "residual_disagree": 0.5944010416666666 }, { "tau": 0.75, "escalated_frac": 0.3841145833333333, "residual_disagree": 0.5944010416666666 }, { "tau": 0.75, "escalated_frac": 0.3841145833333333, "residual_disagree": 0.5944010416666666 }, { "tau": 0.8125, "escalated_frac": 0.4353841145833333, "residual_disagree": 0.5445963541666666 }, { "tau": 0.875, "escalated_frac": 0.4383138020833333, "residual_disagree": 0.5416666666666666 }, { "tau": 0.875, "escalated_frac": 0.4383138020833333, "residual_disagree": 0.5416666666666666 }, { "tau": 0.875, "escalated_frac": 0.4383138020833333, "residual_disagree": 0.5416666666666666 }, { "tau": 0.875, "escalated_frac": 0.4383138020833333, "residual_disagree": 0.5416666666666666 }, { "tau": 0.875, "escalated_frac": 0.4383138020833333, "residual_disagree": 0.5416666666666666 }, { "tau": 0.875, "escalated_frac": 0.4383138020833333, "residual_disagree": 0.5416666666666666 }, { "tau": 0.875, "escalated_frac": 0.4383138020833333, "residual_disagree": 0.5416666666666666 }, { "tau": 0.875, "escalated_frac": 0.4383138020833333, "residual_disagree": 0.5416666666666666 }, { "tau": 0.875, "escalated_frac": 0.4383138020833333, "residual_disagree": 0.5416666666666666 }, { "tau": 0.9579145728643255, "escalated_frac": 0.4874674479166667, "residual_disagree": 0.4931640625 }, { "tau": 1.0, "escalated_frac": 0.4874674479166667, "residual_disagree": 0.4931640625 }, { "tau": 1.0, "escalated_frac": 0.4874674479166667, "residual_disagree": 0.4931640625 }, { "tau": 1.0, "escalated_frac": 0.4874674479166667, "residual_disagree": 0.4931640625 }, { "tau": 1.0, "escalated_frac": 0.4874674479166667, "residual_disagree": 0.4931640625 }, { "tau": 1.0, "escalated_frac": 0.4874674479166667, "residual_disagree": 0.4931640625 }, { "tau": 1.0, "escalated_frac": 0.4874674479166667, "residual_disagree": 0.4931640625 }, { "tau": 1.0, "escalated_frac": 0.4874674479166667, "residual_disagree": 0.4931640625 }, { "tau": 1.0, "escalated_frac": 0.4874674479166667, "residual_disagree": 0.4931640625 }, { "tau": 1.125, "escalated_frac": 0.5319010416666666, "residual_disagree": 0.4495442708333333 }, { "tau": 1.125, "escalated_frac": 0.5319010416666666, "residual_disagree": 0.4495442708333333 }, { "tau": 1.125, "escalated_frac": 0.5319010416666666, "residual_disagree": 0.4495442708333333 }, { "tau": 1.125, "escalated_frac": 0.5319010416666666, "residual_disagree": 0.4495442708333333 }, { "tau": 1.125, "escalated_frac": 0.5319010416666666, "residual_disagree": 0.4495442708333333 }, { "tau": 1.125, "escalated_frac": 0.5319010416666666, "residual_disagree": 0.4495442708333333 }, { "tau": 1.125, "escalated_frac": 0.5319010416666666, "residual_disagree": 0.4495442708333333 }, { "tau": 1.1875, "escalated_frac": 0.56689453125, "residual_disagree": 0.41552734375 }, { "tau": 1.25, "escalated_frac": 0.5686848958333334, "residual_disagree": 0.4137369791666667 }, { "tau": 1.25, "escalated_frac": 0.5686848958333334, "residual_disagree": 0.4137369791666667 }, { "tau": 1.25, "escalated_frac": 0.5686848958333334, "residual_disagree": 0.4137369791666667 }, { "tau": 1.25, "escalated_frac": 0.5686848958333334, "residual_disagree": 0.4137369791666667 }, { "tau": 1.25, "escalated_frac": 0.5686848958333334, "residual_disagree": 0.4137369791666667 }, { "tau": 1.25, "escalated_frac": 0.5686848958333334, "residual_disagree": 0.4137369791666667 }, { "tau": 1.375, "escalated_frac": 0.6022135416666666, "residual_disagree": 0.3810221354166667 }, { "tau": 1.375, "escalated_frac": 0.6022135416666666, "residual_disagree": 0.3810221354166667 }, { "tau": 1.375, "escalated_frac": 0.6022135416666666, "residual_disagree": 0.3810221354166667 }, { "tau": 1.375, "escalated_frac": 0.6022135416666666, "residual_disagree": 0.3810221354166667 }, { "tau": 1.375, "escalated_frac": 0.6022135416666666, "residual_disagree": 0.3810221354166667 }, { "tau": 1.375, "escalated_frac": 0.6022135416666666, "residual_disagree": 0.3810221354166667 }, { "tau": 1.4086055276381728, "escalated_frac": 0.6331380208333334, "residual_disagree": 0.3518880208333333 }, { "tau": 1.5, "escalated_frac": 0.6350911458333334, "residual_disagree": 0.3499348958333333 }, { "tau": 1.5, "escalated_frac": 0.6350911458333334, "residual_disagree": 0.3499348958333333 }, { "tau": 1.5, "escalated_frac": 0.6350911458333334, "residual_disagree": 0.3499348958333333 }, { "tau": 1.5, "escalated_frac": 0.6350911458333334, "residual_disagree": 0.3499348958333333 }, { "tau": 1.5, "escalated_frac": 0.6350911458333334, "residual_disagree": 0.3499348958333333 }, { "tau": 1.5625, "escalated_frac": 0.6629231770833334, "residual_disagree": 0.3229166666666667 }, { "tau": 1.625, "escalated_frac": 0.6640625, "residual_disagree": 0.32177734375 }, { "tau": 1.625, "escalated_frac": 0.6640625, "residual_disagree": 0.32177734375 }, { "tau": 1.625, "escalated_frac": 0.6640625, "residual_disagree": 0.32177734375 }, { "tau": 1.625, "escalated_frac": 0.6640625, "residual_disagree": 0.32177734375 }, { "tau": 1.625, "escalated_frac": 0.6640625, "residual_disagree": 0.32177734375 }, { "tau": 1.625, "escalated_frac": 0.6640625, "residual_disagree": 0.32177734375 }, { "tau": 1.75, "escalated_frac": 0.697265625, "residual_disagree": 0.2898763020833333 }, { "tau": 1.75, "escalated_frac": 0.697265625, "residual_disagree": 0.2898763020833333 }, { "tau": 1.75, "escalated_frac": 0.697265625, "residual_disagree": 0.2898763020833333 }, { "tau": 1.75, "escalated_frac": 0.697265625, "residual_disagree": 0.2898763020833333 }, { "tau": 1.75, "escalated_frac": 0.697265625, "residual_disagree": 0.2898763020833333 }, { "tau": 1.875, "escalated_frac": 0.7234700520833334, "residual_disagree": 0.2646484375 }, { "tau": 1.875, "escalated_frac": 0.7234700520833334, "residual_disagree": 0.2646484375 }, { "tau": 1.875, "escalated_frac": 0.7234700520833334, "residual_disagree": 0.2646484375 }, { "tau": 1.875, "escalated_frac": 0.7234700520833334, "residual_disagree": 0.2646484375 }, { "tau": 1.875, "escalated_frac": 0.7234700520833334, "residual_disagree": 0.2646484375 }, { "tau": 2.0, "escalated_frac": 0.7451171875, "residual_disagree": 0.24348958333333334 }, { "tau": 2.0, "escalated_frac": 0.7451171875, "residual_disagree": 0.24348958333333334 }, { "tau": 2.0, "escalated_frac": 0.7451171875, "residual_disagree": 0.24348958333333334 }, { "tau": 2.0, "escalated_frac": 0.7451171875, "residual_disagree": 0.24348958333333334 }, { "tau": 2.125, "escalated_frac": 0.76708984375, "residual_disagree": 0.22184244791666666 }, { "tau": 2.125, "escalated_frac": 0.76708984375, "residual_disagree": 0.22184244791666666 }, { "tau": 2.125, "escalated_frac": 0.76708984375, "residual_disagree": 0.22184244791666666 }, { "tau": 2.125, "escalated_frac": 0.76708984375, "residual_disagree": 0.22184244791666666 }, { "tau": 2.25, "escalated_frac": 0.787109375, "residual_disagree": 0.20263671875 }, { "tau": 2.25, "escalated_frac": 0.787109375, "residual_disagree": 0.20263671875 }, { "tau": 2.25, "escalated_frac": 0.787109375, "residual_disagree": 0.20263671875 }, { "tau": 2.375, "escalated_frac": 0.8030598958333334, "residual_disagree": 0.18701171875 }, { "tau": 2.375, "escalated_frac": 0.8030598958333334, "residual_disagree": 0.18701171875 }, { "tau": 2.375, "escalated_frac": 0.8030598958333334, "residual_disagree": 0.18701171875 }, { "tau": 2.375, "escalated_frac": 0.8030598958333334, "residual_disagree": 0.18701171875 }, { "tau": 2.5, "escalated_frac": 0.8204752604166666, "residual_disagree": 0.169921875 }, { "tau": 2.5, "escalated_frac": 0.8204752604166666, "residual_disagree": 0.169921875 }, { "tau": 2.5, "escalated_frac": 0.8204752604166666, "residual_disagree": 0.169921875 }, { "tau": 2.625, "escalated_frac": 0.8385416666666666, "residual_disagree": 0.15234375 }, { "tau": 2.625, "escalated_frac": 0.8385416666666666, "residual_disagree": 0.15234375 }, { "tau": 2.625, "escalated_frac": 0.8385416666666666, "residual_disagree": 0.15234375 }, { "tau": 2.75, "escalated_frac": 0.853515625, "residual_disagree": 0.13736979166666666 }, { "tau": 2.75, "escalated_frac": 0.853515625, "residual_disagree": 0.13736979166666666 }, { "tau": 2.75, "escalated_frac": 0.853515625, "residual_disagree": 0.13736979166666666 }, { "tau": 2.875, "escalated_frac": 0.86669921875, "residual_disagree": 0.12483723958333333 }, { "tau": 2.875, "escalated_frac": 0.86669921875, "residual_disagree": 0.12483723958333333 }, { "tau": 2.875, "escalated_frac": 0.86669921875, "residual_disagree": 0.12483723958333333 }, { "tau": 3.0, "escalated_frac": 0.88134765625, "residual_disagree": 0.11051432291666667 }, { "tau": 3.0, "escalated_frac": 0.88134765625, "residual_disagree": 0.11051432291666667 }, { "tau": 3.125, "escalated_frac": 0.8942057291666666, "residual_disagree": 0.09814453125 }, { "tau": 3.125, "escalated_frac": 0.8942057291666666, "residual_disagree": 0.09814453125 }, { "tau": 3.217650753768851, "escalated_frac": 0.9044596354166666, "residual_disagree": 0.08837890625 }, { "tau": 3.25, "escalated_frac": 0.9044596354166666, "residual_disagree": 0.08837890625 }, { "tau": 3.375, "escalated_frac": 0.9130859375, "residual_disagree": 0.08024088541666667 }, { "tau": 3.375, "escalated_frac": 0.9130859375, "residual_disagree": 0.08024088541666667 }, { "tau": 3.5, "escalated_frac": 0.92041015625, "residual_disagree": 0.07307942708333333 }, { "tau": 3.625, "escalated_frac": 0.9267578125, "residual_disagree": 0.06722005208333333 }, { "tau": 3.75, "escalated_frac": 0.93310546875, "residual_disagree": 0.0615234375 }, { "tau": 3.75, "escalated_frac": 0.93310546875, "residual_disagree": 0.0615234375 }, { "tau": 3.875, "escalated_frac": 0.9401041666666666, "residual_disagree": 0.054850260416666664 }, { "tau": 4.0, "escalated_frac": 0.9456380208333334, "residual_disagree": 0.049641927083333336 }, { "tau": 4.125, "escalated_frac": 0.95166015625, "residual_disagree": 0.0439453125 }, { "tau": 4.25, "escalated_frac": 0.9560546875, "residual_disagree": 0.039876302083333336 }, { "tau": 4.5, "escalated_frac": 0.9637044270833334, "residual_disagree": 0.032552083333333336 }, { "tau": 4.75, "escalated_frac": 0.9690755208333334, "residual_disagree": 0.02734375 }, { "tau": 5.0, "escalated_frac": 0.974609375, "residual_disagree": 0.022298177083333332 }, { "tau": 5.25, "escalated_frac": 0.97802734375, "residual_disagree": 0.01904296875 }, { "tau": 5.75, "escalated_frac": 0.9847005208333334, "residual_disagree": 0.01318359375 }, { "tau": 6.25, "escalated_frac": 0.98974609375, "residual_disagree": 0.008463541666666666 }, { "tau": 7.25, "escalated_frac": 0.9947916666666666, "residual_disagree": 0.004069010416666667 }, { "tau": 14.875, "escalated_frac": 0.9998372395833334, "residual_disagree": 0.00016276041666666666 } ], "per_domain": { "code": { "agreement_rate": 0.017578125, "auroc": 0.5606914071128782 }, "math": { "agreement_rate": 0.0546875, "auroc": 0.6518410566706021 }, "chat": { "agreement_rate": 0.0205078125, "auroc": 0.5552153064615677 }, "french": { "agreement_rate": 0.0263671875, "auroc": 0.6100338051190609 }, "english": { "agreement_rate": 0.029296875, "auroc": 0.6428571428571429 }, "reasoning": { "agreement_rate": 0.029296875, "auroc": 0.5358484238765929 } }, "escalation_frac_for_99pct": 0.98974609375 }, "3": { "bits": 3, "n_positions": 6144, "agreement_rate": 0.73095703125, "mean_kl_ref_q": 0.7811363339424133, "auroc_margin_predicts_disagreement": 0.8521382349292252, "median_margin_agree": 5.25, "median_margin_disagree": 1.125, "escalation_curve": [ { "tau": 0.0, "escalated_frac": 0.0, "residual_disagree": 0.26904296875 }, { "tau": 0.0, "escalated_frac": 0.0, "residual_disagree": 0.26904296875 }, { "tau": 0.0, "escalated_frac": 0.0, "residual_disagree": 0.26904296875 }, { "tau": 0.0, "escalated_frac": 0.0, "residual_disagree": 0.26904296875 }, { "tau": 0.0, "escalated_frac": 0.0, "residual_disagree": 0.26904296875 }, { "tau": 0.125, "escalated_frac": 0.020833333333333332, "residual_disagree": 0.2550455729166667 }, { "tau": 0.125, "escalated_frac": 0.020833333333333332, "residual_disagree": 0.2550455729166667 }, { "tau": 0.125, "escalated_frac": 0.020833333333333332, "residual_disagree": 0.2550455729166667 }, { "tau": 0.25, "escalated_frac": 0.038248697916666664, "residual_disagree": 0.2421875 }, { "tau": 0.25, "escalated_frac": 0.038248697916666664, "residual_disagree": 0.2421875 }, { "tau": 0.25, "escalated_frac": 0.038248697916666664, "residual_disagree": 0.2421875 }, { "tau": 0.25, "escalated_frac": 0.038248697916666664, "residual_disagree": 0.2421875 }, { "tau": 0.25, "escalated_frac": 0.038248697916666664, "residual_disagree": 0.2421875 }, { "tau": 0.25, "escalated_frac": 0.038248697916666664, "residual_disagree": 0.2421875 }, { "tau": 0.25, "escalated_frac": 0.038248697916666664, "residual_disagree": 0.2421875 }, { "tau": 0.375, "escalated_frac": 0.07096354166666667, "residual_disagree": 0.21988932291666666 }, { "tau": 0.375, "escalated_frac": 0.07096354166666667, "residual_disagree": 0.21988932291666666 }, { "tau": 0.375, "escalated_frac": 0.07096354166666667, "residual_disagree": 0.21988932291666666 }, { "tau": 0.5, "escalated_frac": 0.08723958333333333, "residual_disagree": 0.20930989583333334 }, { "tau": 0.5, "escalated_frac": 0.08723958333333333, "residual_disagree": 0.20930989583333334 }, { "tau": 0.5, "escalated_frac": 0.08723958333333333, "residual_disagree": 0.20930989583333334 }, { "tau": 0.5, "escalated_frac": 0.08723958333333333, "residual_disagree": 0.20930989583333334 }, { "tau": 0.5, "escalated_frac": 0.08723958333333333, "residual_disagree": 0.20930989583333334 }, { "tau": 0.5, "escalated_frac": 0.08723958333333333, "residual_disagree": 0.20930989583333334 }, { "tau": 0.625, "escalated_frac": 0.11946614583333333, "residual_disagree": 0.1904296875 }, { "tau": 0.625, "escalated_frac": 0.11946614583333333, "residual_disagree": 0.1904296875 }, { "tau": 0.625, "escalated_frac": 0.11946614583333333, "residual_disagree": 0.1904296875 }, { "tau": 0.625, "escalated_frac": 0.11946614583333333, "residual_disagree": 0.1904296875 }, { "tau": 0.75, "escalated_frac": 0.13688151041666666, "residual_disagree": 0.18001302083333334 }, { "tau": 0.75, "escalated_frac": 0.13688151041666666, "residual_disagree": 0.18001302083333334 }, { "tau": 0.75, "escalated_frac": 0.13688151041666666, "residual_disagree": 0.18001302083333334 }, { "tau": 0.75, "escalated_frac": 0.13688151041666666, "residual_disagree": 0.18001302083333334 }, { "tau": 0.75, "escalated_frac": 0.13688151041666666, "residual_disagree": 0.18001302083333334 }, { "tau": 0.875, "escalated_frac": 0.16552734375, "residual_disagree": 0.16162109375 }, { "tau": 0.875, "escalated_frac": 0.16552734375, "residual_disagree": 0.16162109375 }, { "tau": 0.875, "escalated_frac": 0.16552734375, "residual_disagree": 0.16162109375 }, { "tau": 0.875, "escalated_frac": 0.16552734375, "residual_disagree": 0.16162109375 }, { "tau": 1.0, "escalated_frac": 0.18131510416666666, "residual_disagree": 0.15185546875 }, { "tau": 1.0, "escalated_frac": 0.18131510416666666, "residual_disagree": 0.15185546875 }, { "tau": 1.0, "escalated_frac": 0.18131510416666666, "residual_disagree": 0.15185546875 }, { "tau": 1.0, "escalated_frac": 0.18131510416666666, "residual_disagree": 0.15185546875 }, { "tau": 1.0, "escalated_frac": 0.18131510416666666, "residual_disagree": 0.15185546875 }, { "tau": 1.0, "escalated_frac": 0.18131510416666666, "residual_disagree": 0.15185546875 }, { "tau": 1.125, "escalated_frac": 0.21223958333333334, "residual_disagree": 0.13525390625 }, { "tau": 1.125, "escalated_frac": 0.21223958333333334, "residual_disagree": 0.13525390625 }, { "tau": 1.25, "escalated_frac": 0.22574869791666666, "residual_disagree": 0.12890625 }, { "tau": 1.25, "escalated_frac": 0.22574869791666666, "residual_disagree": 0.12890625 }, { "tau": 1.25, "escalated_frac": 0.22574869791666666, "residual_disagree": 0.12890625 }, { "tau": 1.25, "escalated_frac": 0.22574869791666666, "residual_disagree": 0.12890625 }, { "tau": 1.25, "escalated_frac": 0.22574869791666666, "residual_disagree": 0.12890625 }, { "tau": 1.25, "escalated_frac": 0.22574869791666666, "residual_disagree": 0.12890625 }, { "tau": 1.375, "escalated_frac": 0.2521158854166667, "residual_disagree": 0.11555989583333333 }, { "tau": 1.375, "escalated_frac": 0.2521158854166667, "residual_disagree": 0.11555989583333333 }, { "tau": 1.5, "escalated_frac": 0.2635091145833333, "residual_disagree": 0.10807291666666667 }, { "tau": 1.5, "escalated_frac": 0.2635091145833333, "residual_disagree": 0.10807291666666667 }, { "tau": 1.5, "escalated_frac": 0.2635091145833333, "residual_disagree": 0.10807291666666667 }, { "tau": 1.5, "escalated_frac": 0.2635091145833333, "residual_disagree": 0.10807291666666667 }, { "tau": 1.625, "escalated_frac": 0.28466796875, "residual_disagree": 0.09733072916666667 }, { "tau": 1.625, "escalated_frac": 0.28466796875, "residual_disagree": 0.09733072916666667 }, { "tau": 1.625, "escalated_frac": 0.28466796875, "residual_disagree": 0.09733072916666667 }, { "tau": 1.75, "escalated_frac": 0.2985026041666667, "residual_disagree": 0.09000651041666667 }, { "tau": 1.75, "escalated_frac": 0.2985026041666667, "residual_disagree": 0.09000651041666667 }, { "tau": 1.75, "escalated_frac": 0.2985026041666667, "residual_disagree": 0.09000651041666667 }, { "tau": 1.75, "escalated_frac": 0.2985026041666667, "residual_disagree": 0.09000651041666667 }, { "tau": 1.75, "escalated_frac": 0.2985026041666667, "residual_disagree": 0.09000651041666667 }, { "tau": 1.875, "escalated_frac": 0.32177734375, "residual_disagree": 0.07926432291666667 }, { "tau": 1.875, "escalated_frac": 0.32177734375, "residual_disagree": 0.07926432291666667 }, { "tau": 2.0, "escalated_frac": 0.33203125, "residual_disagree": 0.0751953125 }, { "tau": 2.0, "escalated_frac": 0.33203125, "residual_disagree": 0.0751953125 }, { "tau": 2.0, "escalated_frac": 0.33203125, "residual_disagree": 0.0751953125 }, { "tau": 2.0, "escalated_frac": 0.33203125, "residual_disagree": 0.0751953125 }, { "tau": 2.125, "escalated_frac": 0.3527018229166667, "residual_disagree": 0.068359375 }, { "tau": 2.125, "escalated_frac": 0.3527018229166667, "residual_disagree": 0.068359375 }, { "tau": 2.25, "escalated_frac": 0.3619791666666667, "residual_disagree": 0.06461588541666667 }, { "tau": 2.25, "escalated_frac": 0.3619791666666667, "residual_disagree": 0.06461588541666667 }, { "tau": 2.25, "escalated_frac": 0.3619791666666667, "residual_disagree": 0.06461588541666667 }, { "tau": 2.375, "escalated_frac": 0.3811848958333333, "residual_disagree": 0.057942708333333336 }, { "tau": 2.375, "escalated_frac": 0.3811848958333333, "residual_disagree": 0.057942708333333336 }, { "tau": 2.5, "escalated_frac": 0.3907877604166667, "residual_disagree": 0.0546875 }, { "tau": 2.5, "escalated_frac": 0.3907877604166667, "residual_disagree": 0.0546875 }, { "tau": 2.5, "escalated_frac": 0.3907877604166667, "residual_disagree": 0.0546875 }, { "tau": 2.5, "escalated_frac": 0.3907877604166667, "residual_disagree": 0.0546875 }, { "tau": 2.625, "escalated_frac": 0.4083658854166667, "residual_disagree": 0.04931640625 }, { "tau": 2.75, "escalated_frac": 0.41650390625, "residual_disagree": 0.046549479166666664 }, { "tau": 2.75, "escalated_frac": 0.41650390625, "residual_disagree": 0.046549479166666664 }, { "tau": 2.75, "escalated_frac": 0.41650390625, "residual_disagree": 0.046549479166666664 }, { "tau": 2.75, "escalated_frac": 0.41650390625, "residual_disagree": 0.046549479166666664 }, { "tau": 2.875, "escalated_frac": 0.4334309895833333, "residual_disagree": 0.040201822916666664 }, { "tau": 3.0, "escalated_frac": 0.4404296875, "residual_disagree": 0.038248697916666664 }, { "tau": 3.0, "escalated_frac": 0.4404296875, "residual_disagree": 0.038248697916666664 }, { "tau": 3.0, "escalated_frac": 0.4404296875, "residual_disagree": 0.038248697916666664 }, { "tau": 3.0, "escalated_frac": 0.4404296875, "residual_disagree": 0.038248697916666664 }, { "tau": 3.125, "escalated_frac": 0.4591471354166667, "residual_disagree": 0.034016927083333336 }, { "tau": 3.125, "escalated_frac": 0.4591471354166667, "residual_disagree": 0.034016927083333336 }, { "tau": 3.25, "escalated_frac": 0.4685872395833333, "residual_disagree": 0.031575520833333336 }, { "tau": 3.25, "escalated_frac": 0.4685872395833333, "residual_disagree": 0.031575520833333336 }, { "tau": 3.25, "escalated_frac": 0.4685872395833333, "residual_disagree": 0.031575520833333336 }, { "tau": 3.375, "escalated_frac": 0.4855143229166667, "residual_disagree": 0.026529947916666668 }, { "tau": 3.375, "escalated_frac": 0.4855143229166667, "residual_disagree": 0.026529947916666668 }, { "tau": 3.5, "escalated_frac": 0.4947916666666667, "residual_disagree": 0.024251302083333332 }, { "tau": 3.5, "escalated_frac": 0.4947916666666667, "residual_disagree": 0.024251302083333332 }, { "tau": 3.5, "escalated_frac": 0.4947916666666667, "residual_disagree": 0.024251302083333332 }, { "tau": 3.625, "escalated_frac": 0.5110677083333334, "residual_disagree": 0.021321614583333332 }, { "tau": 3.75, "escalated_frac": 0.51708984375, "residual_disagree": 0.02001953125 }, { "tau": 3.75, "escalated_frac": 0.51708984375, "residual_disagree": 0.02001953125 }, { "tau": 3.75, "escalated_frac": 0.51708984375, "residual_disagree": 0.02001953125 }, { "tau": 3.875, "escalated_frac": 0.53173828125, "residual_disagree": 0.017415364583333332 }, { "tau": 3.875, "escalated_frac": 0.53173828125, "residual_disagree": 0.017415364583333332 }, { "tau": 4.0, "escalated_frac": 0.5384114583333334, "residual_disagree": 0.016927083333333332 }, { "tau": 4.0, "escalated_frac": 0.5384114583333334, "residual_disagree": 0.016927083333333332 }, { "tau": 4.125, "escalated_frac": 0.5511067708333334, "residual_disagree": 0.01416015625 }, { "tau": 4.25, "escalated_frac": 0.5568033854166666, "residual_disagree": 0.013997395833333334 }, { "tau": 4.25, "escalated_frac": 0.5568033854166666, "residual_disagree": 0.013997395833333334 }, { "tau": 4.25, "escalated_frac": 0.5568033854166666, "residual_disagree": 0.013997395833333334 }, { "tau": 4.375, "escalated_frac": 0.5690104166666666, "residual_disagree": 0.01318359375 }, { "tau": 4.5, "escalated_frac": 0.5745442708333334, "residual_disagree": 0.012858072916666666 }, { "tau": 4.5, "escalated_frac": 0.5745442708333334, "residual_disagree": 0.012858072916666666 }, { "tau": 4.625, "escalated_frac": 0.5865885416666666, "residual_disagree": 0.011393229166666666 }, { "tau": 4.75, "escalated_frac": 0.5926106770833334, "residual_disagree": 0.010904947916666666 }, { "tau": 4.75, "escalated_frac": 0.5926106770833334, "residual_disagree": 0.010904947916666666 }, { "tau": 4.875, "escalated_frac": 0.6017252604166666, "residual_disagree": 0.009928385416666666 }, { "tau": 5.0, "escalated_frac": 0.6067708333333334, "residual_disagree": 0.00927734375 }, { "tau": 5.0, "escalated_frac": 0.6067708333333334, "residual_disagree": 0.00927734375 }, { "tau": 5.125, "escalated_frac": 0.61767578125, "residual_disagree": 0.0087890625 }, { "tau": 5.25, "escalated_frac": 0.6214192708333334, "residual_disagree": 0.008626302083333334 }, { "tau": 5.25, "escalated_frac": 0.6214192708333334, "residual_disagree": 0.008626302083333334 }, { "tau": 5.375, "escalated_frac": 0.63134765625, "residual_disagree": 0.00732421875 }, { "tau": 5.5, "escalated_frac": 0.6359049479166666, "residual_disagree": 0.0068359375 }, { "tau": 5.5, "escalated_frac": 0.6359049479166666, "residual_disagree": 0.0068359375 }, { "tau": 5.75, "escalated_frac": 0.6466471354166666, "residual_disagree": 0.006510416666666667 }, { "tau": 5.75, "escalated_frac": 0.6466471354166666, "residual_disagree": 0.006510416666666667 }, { "tau": 6.0, "escalated_frac": 0.6573893229166666, "residual_disagree": 0.00634765625 }, { "tau": 6.0, "escalated_frac": 0.6573893229166666, "residual_disagree": 0.00634765625 }, { "tau": 6.25, "escalated_frac": 0.6661783854166666, "residual_disagree": 0.005208333333333333 }, { "tau": 6.375, "escalated_frac": 0.67236328125, "residual_disagree": 0.004720052083333333 }, { "tau": 6.5, "escalated_frac": 0.6748046875, "residual_disagree": 0.004720052083333333 }, { "tau": 6.625, "escalated_frac": 0.6818033854166666, "residual_disagree": 0.004069010416666667 }, { "tau": 6.75, "escalated_frac": 0.6863606770833334, "residual_disagree": 0.00390625 }, { "tau": 6.871231155778901, "escalated_frac": 0.693359375, "residual_disagree": 0.0035807291666666665 }, { "tau": 7.0, "escalated_frac": 0.6964518229166666, "residual_disagree": 0.0035807291666666665 }, { "tau": 7.125, "escalated_frac": 0.7024739583333334, "residual_disagree": 0.0032552083333333335 }, { "tau": 7.25, "escalated_frac": 0.7052408854166666, "residual_disagree": 0.0030924479166666665 }, { "tau": 7.5, "escalated_frac": 0.71337890625, "residual_disagree": 0.0027669270833333335 }, { "tau": 7.5, "escalated_frac": 0.71337890625, "residual_disagree": 0.0027669270833333335 }, { "tau": 7.75, "escalated_frac": 0.7233072916666666, "residual_disagree": 0.00244140625 }, { "tau": 7.75, "escalated_frac": 0.7233072916666666, "residual_disagree": 0.00244140625 }, { "tau": 7.875, "escalated_frac": 0.7314453125, "residual_disagree": 0.0022786458333333335 }, { "tau": 8.125, "escalated_frac": 0.7373046875, "residual_disagree": 0.0022786458333333335 }, { "tau": 8.25, "escalated_frac": 0.73974609375, "residual_disagree": 0.0022786458333333335 }, { "tau": 8.5, "escalated_frac": 0.7473958333333334, "residual_disagree": 0.0021158854166666665 }, { "tau": 8.75, "escalated_frac": 0.7535807291666666, "residual_disagree": 0.0017903645833333333 }, { "tau": 8.875, "escalated_frac": 0.7576497395833334, "residual_disagree": 0.0017903645833333333 }, { "tau": 9.0, "escalated_frac": 0.7600911458333334, "residual_disagree": 0.0017903645833333333 }, { "tau": 9.25, "escalated_frac": 0.7669270833333334, "residual_disagree": 0.0017903645833333333 }, { "tau": 9.5, "escalated_frac": 0.7736002604166666, "residual_disagree": 0.0017903645833333333 }, { "tau": 9.5, "escalated_frac": 0.7736002604166666, "residual_disagree": 0.0017903645833333333 }, { "tau": 9.75, "escalated_frac": 0.78076171875, "residual_disagree": 0.0016276041666666667 }, { "tau": 10.0, "escalated_frac": 0.7862955729166666, "residual_disagree": 0.00146484375 }, { "tau": 10.25, "escalated_frac": 0.7926432291666666, "residual_disagree": 0.00146484375 }, { "tau": 10.375, "escalated_frac": 0.7967122395833334, "residual_disagree": 0.0013020833333333333 }, { "tau": 10.625, "escalated_frac": 0.8035481770833334, "residual_disagree": 0.0013020833333333333 }, { "tau": 10.75, "escalated_frac": 0.8059895833333334, "residual_disagree": 0.0013020833333333333 }, { "tau": 10.979271356783897, "escalated_frac": 0.81396484375, "residual_disagree": 0.0009765625 }, { "tau": 11.125, "escalated_frac": 0.81787109375, "residual_disagree": 0.0009765625 }, { "tau": 11.25, "escalated_frac": 0.8203125, "residual_disagree": 0.0009765625 }, { "tau": 11.5, "escalated_frac": 0.8269856770833334, "residual_disagree": 0.0009765625 }, { "tau": 11.75, "escalated_frac": 0.8336588541666666, "residual_disagree": 0.0009765625 }, { "tau": 11.875, "escalated_frac": 0.8369140625, "residual_disagree": 0.0009765625 }, { "tau": 12.125, "escalated_frac": 0.8430989583333334, "residual_disagree": 0.0008138020833333334 }, { "tau": 12.364949748743697, "escalated_frac": 0.84912109375, "residual_disagree": 0.0006510416666666666 }, { "tau": 12.5, "escalated_frac": 0.8518880208333334, "residual_disagree": 0.0006510416666666666 }, { "tau": 12.75, "escalated_frac": 0.8583984375, "residual_disagree": 0.00048828125 }, { "tau": 13.0, "escalated_frac": 0.8640950520833334, "residual_disagree": 0.0003255208333333333 }, { "tau": 13.25, "escalated_frac": 0.8684895833333334, "residual_disagree": 0.0003255208333333333 }, { "tau": 13.5, "escalated_frac": 0.87353515625, "residual_disagree": 0.00016276041666666666 }, { "tau": 13.75, "escalated_frac": 0.8790690104166666, "residual_disagree": 0.0 }, { "tau": 13.875, "escalated_frac": 0.8834635416666666, "residual_disagree": 0.0 }, { "tau": 14.125, "escalated_frac": 0.8885091145833334, "residual_disagree": 0.0 }, { "tau": 14.25, "escalated_frac": 0.8917643229166666, "residual_disagree": 0.0 }, { "tau": 14.5, "escalated_frac": 0.8963216145833334, "residual_disagree": 0.0 }, { "tau": 14.75, "escalated_frac": 0.9021809895833334, "residual_disagree": 0.0 }, { "tau": 15.0, "escalated_frac": 0.9075520833333334, "residual_disagree": 0.0 }, { "tau": 15.27763819095469, "escalated_frac": 0.91455078125, "residual_disagree": 0.0 }, { "tau": 15.625, "escalated_frac": 0.9189453125, "residual_disagree": 0.0 }, { "tau": 16.0, "escalated_frac": 0.9236653645833334, "residual_disagree": 0.0 }, { "tau": 16.25, "escalated_frac": 0.92822265625, "residual_disagree": 0.0 }, { "tau": 16.75, "escalated_frac": 0.9339192708333334, "residual_disagree": 0.0 }, { "tau": 17.25, "escalated_frac": 0.9392903645833334, "residual_disagree": 0.0 }, { "tau": 17.625, "escalated_frac": 0.9440104166666666, "residual_disagree": 0.0 }, { "tau": 17.875, "escalated_frac": 0.9484049479166666, "residual_disagree": 0.0 }, { "tau": 18.375, "escalated_frac": 0.9542643229166666, "residual_disagree": 0.0 }, { "tau": 18.875, "escalated_frac": 0.95947265625, "residual_disagree": 0.0 }, { "tau": 19.375, "escalated_frac": 0.9640299479166666, "residual_disagree": 0.0 }, { "tau": 20.0, "escalated_frac": 0.9695638020833334, "residual_disagree": 0.0 }, { "tau": 20.375, "escalated_frac": 0.97412109375, "residual_disagree": 0.0 }, { "tau": 21.125, "escalated_frac": 0.9793294270833334, "residual_disagree": 0.0 }, { "tau": 22.173994974874404, "escalated_frac": 0.98486328125, "residual_disagree": 0.0 }, { "tau": 23.53266331658301, "escalated_frac": 0.9899088541666666, "residual_disagree": 0.0 }, { "tau": 25.391331658291392, "escalated_frac": 0.9949544270833334, "residual_disagree": 0.0 }, { "tau": 32.375, "escalated_frac": 0.9998372395833334, "residual_disagree": 0.0 } ], "per_domain": { "code": { "agreement_rate": 0.728515625, "auroc": 0.8375484598916041 }, "math": { "agreement_rate": 0.8193359375, "auroc": 0.8957188416068035 }, "chat": { "agreement_rate": 0.6669921875, "auroc": 0.8009257072686913 }, "french": { "agreement_rate": 0.728515625, "auroc": 0.8465605531660463 }, "english": { "agreement_rate": 0.6875, "auroc": 0.8543834339488636 }, "reasoning": { "agreement_rate": 0.7548828125, "auroc": 0.8626838055282106 } }, "escalation_frac_for_99pct": 0.6017252604166666 }, "4": { "bits": 4, "n_positions": 6144, "agreement_rate": 0.8740234375, "mean_kl_ref_q": 0.19848966598510742, "auroc_margin_predicts_disagreement": 0.8982447466304814, "median_margin_agree": 6.5, "median_margin_disagree": 0.75, "escalation_curve": [ { "tau": 0.0, "escalated_frac": 0.0, "residual_disagree": 0.1259765625 }, { "tau": 0.0, "escalated_frac": 0.0, "residual_disagree": 0.1259765625 }, { "tau": 0.0, "escalated_frac": 0.0, "residual_disagree": 0.1259765625 }, { "tau": 0.125, "escalated_frac": 0.014485677083333334, "residual_disagree": 0.11783854166666667 }, { "tau": 0.125, "escalated_frac": 0.014485677083333334, "residual_disagree": 0.11783854166666667 }, { "tau": 0.125, "escalated_frac": 0.014485677083333334, "residual_disagree": 0.11783854166666667 }, { "tau": 0.25, "escalated_frac": 0.0263671875, "residual_disagree": 0.1083984375 }, { "tau": 0.25, "escalated_frac": 0.0263671875, "residual_disagree": 0.1083984375 }, { "tau": 0.25, "escalated_frac": 0.0263671875, "residual_disagree": 0.1083984375 }, { "tau": 0.25, "escalated_frac": 0.0263671875, "residual_disagree": 0.1083984375 }, { "tau": 0.25, "escalated_frac": 0.0263671875, "residual_disagree": 0.1083984375 }, { "tau": 0.375, "escalated_frac": 0.053059895833333336, "residual_disagree": 0.09309895833333333 }, { "tau": 0.375, "escalated_frac": 0.053059895833333336, "residual_disagree": 0.09309895833333333 }, { "tau": 0.5, "escalated_frac": 0.06266276041666667, "residual_disagree": 0.08756510416666667 }, { "tau": 0.5, "escalated_frac": 0.06266276041666667, "residual_disagree": 0.08756510416666667 }, { "tau": 0.5, "escalated_frac": 0.06266276041666667, "residual_disagree": 0.08756510416666667 }, { "tau": 0.5, "escalated_frac": 0.06266276041666667, "residual_disagree": 0.08756510416666667 }, { "tau": 0.5, "escalated_frac": 0.06266276041666667, "residual_disagree": 0.08756510416666667 }, { "tau": 0.625, "escalated_frac": 0.08821614583333333, "residual_disagree": 0.07682291666666667 }, { "tau": 0.625, "escalated_frac": 0.08821614583333333, "residual_disagree": 0.07682291666666667 }, { "tau": 0.75, "escalated_frac": 0.09781901041666667, "residual_disagree": 0.07291666666666667 }, { "tau": 0.75, "escalated_frac": 0.09781901041666667, "residual_disagree": 0.07291666666666667 }, { "tau": 0.75, "escalated_frac": 0.09781901041666667, "residual_disagree": 0.07291666666666667 }, { "tau": 0.75, "escalated_frac": 0.09781901041666667, "residual_disagree": 0.07291666666666667 }, { "tau": 0.75, "escalated_frac": 0.09781901041666667, "residual_disagree": 0.07291666666666667 }, { "tau": 0.875, "escalated_frac": 0.12337239583333333, "residual_disagree": 0.062174479166666664 }, { "tau": 0.875, "escalated_frac": 0.12337239583333333, "residual_disagree": 0.062174479166666664 }, { "tau": 1.0, "escalated_frac": 0.1318359375, "residual_disagree": 0.05859375 }, { "tau": 1.0, "escalated_frac": 0.1318359375, "residual_disagree": 0.05859375 }, { "tau": 1.0, "escalated_frac": 0.1318359375, "residual_disagree": 0.05859375 }, { "tau": 1.0, "escalated_frac": 0.1318359375, "residual_disagree": 0.05859375 }, { "tau": 1.125, "escalated_frac": 0.15543619791666666, "residual_disagree": 0.050455729166666664 }, { "tau": 1.125, "escalated_frac": 0.15543619791666666, "residual_disagree": 0.050455729166666664 }, { "tau": 1.25, "escalated_frac": 0.16292317708333334, "residual_disagree": 0.047526041666666664 }, { "tau": 1.25, "escalated_frac": 0.16292317708333334, "residual_disagree": 0.047526041666666664 }, { "tau": 1.25, "escalated_frac": 0.16292317708333334, "residual_disagree": 0.047526041666666664 }, { "tau": 1.25, "escalated_frac": 0.16292317708333334, "residual_disagree": 0.047526041666666664 }, { "tau": 1.25, "escalated_frac": 0.16292317708333334, "residual_disagree": 0.047526041666666664 }, { "tau": 1.375, "escalated_frac": 0.18880208333333334, "residual_disagree": 0.040364583333333336 }, { "tau": 1.375, "escalated_frac": 0.18880208333333334, "residual_disagree": 0.040364583333333336 }, { "tau": 1.5, "escalated_frac": 0.19612630208333334, "residual_disagree": 0.0390625 }, { "tau": 1.5, "escalated_frac": 0.19612630208333334, "residual_disagree": 0.0390625 }, { "tau": 1.5, "escalated_frac": 0.19612630208333334, "residual_disagree": 0.0390625 }, { "tau": 1.5, "escalated_frac": 0.19612630208333334, "residual_disagree": 0.0390625 }, { "tau": 1.625, "escalated_frac": 0.21663411458333334, "residual_disagree": 0.03271484375 }, { "tau": 1.75, "escalated_frac": 0.22412109375, "residual_disagree": 0.030436197916666668 }, { "tau": 1.75, "escalated_frac": 0.22412109375, "residual_disagree": 0.030436197916666668 }, { "tau": 1.75, "escalated_frac": 0.22412109375, "residual_disagree": 0.030436197916666668 }, { "tau": 1.75, "escalated_frac": 0.22412109375, "residual_disagree": 0.030436197916666668 }, { "tau": 1.875, "escalated_frac": 0.24235026041666666, "residual_disagree": 0.027180989583333332 }, { "tau": 2.0, "escalated_frac": 0.24886067708333334, "residual_disagree": 0.026041666666666668 }, { "tau": 2.0, "escalated_frac": 0.24886067708333334, "residual_disagree": 0.026041666666666668 }, { "tau": 2.0, "escalated_frac": 0.24886067708333334, "residual_disagree": 0.026041666666666668 }, { "tau": 2.0, "escalated_frac": 0.24886067708333334, "residual_disagree": 0.026041666666666668 }, { "tau": 2.125, "escalated_frac": 0.2688802083333333, "residual_disagree": 0.02197265625 }, { "tau": 2.125, "escalated_frac": 0.2688802083333333, "residual_disagree": 0.02197265625 }, { "tau": 2.25, "escalated_frac": 0.2766927083333333, "residual_disagree": 0.020182291666666668 }, { "tau": 2.25, "escalated_frac": 0.2766927083333333, "residual_disagree": 0.020182291666666668 }, { "tau": 2.25, "escalated_frac": 0.2766927083333333, "residual_disagree": 0.020182291666666668 }, { "tau": 2.375, "escalated_frac": 0.2939453125, "residual_disagree": 0.017903645833333332 }, { "tau": 2.5, "escalated_frac": 0.3004557291666667, "residual_disagree": 0.017252604166666668 }, { "tau": 2.5, "escalated_frac": 0.3004557291666667, "residual_disagree": 0.017252604166666668 }, { "tau": 2.5, "escalated_frac": 0.3004557291666667, "residual_disagree": 0.017252604166666668 }, { "tau": 2.5, "escalated_frac": 0.3004557291666667, "residual_disagree": 0.017252604166666668 }, { "tau": 2.625, "escalated_frac": 0.3196614583333333, "residual_disagree": 0.014811197916666666 }, { "tau": 2.75, "escalated_frac": 0.3263346354166667, "residual_disagree": 0.013834635416666666 }, { "tau": 2.75, "escalated_frac": 0.3263346354166667, "residual_disagree": 0.013834635416666666 }, { "tau": 2.75, "escalated_frac": 0.3263346354166667, "residual_disagree": 0.013834635416666666 }, { "tau": 2.75, "escalated_frac": 0.3263346354166667, "residual_disagree": 0.013834635416666666 }, { "tau": 2.875, "escalated_frac": 0.3448893229166667, "residual_disagree": 0.01123046875 }, { "tau": 3.0, "escalated_frac": 0.3512369791666667, "residual_disagree": 0.01025390625 }, { "tau": 3.0, "escalated_frac": 0.3512369791666667, "residual_disagree": 0.01025390625 }, { "tau": 3.0, "escalated_frac": 0.3512369791666667, "residual_disagree": 0.01025390625 }, { "tau": 3.125, "escalated_frac": 0.3658854166666667, "residual_disagree": 0.009114583333333334 }, { "tau": 3.25, "escalated_frac": 0.37060546875, "residual_disagree": 0.00830078125 }, { "tau": 3.25, "escalated_frac": 0.37060546875, "residual_disagree": 0.00830078125 }, { "tau": 3.25, "escalated_frac": 0.37060546875, "residual_disagree": 0.00830078125 }, { "tau": 3.375, "escalated_frac": 0.38623046875, "residual_disagree": 0.007161458333333333 }, { "tau": 3.5, "escalated_frac": 0.3914388020833333, "residual_disagree": 0.006673177083333333 }, { "tau": 3.5, "escalated_frac": 0.3914388020833333, "residual_disagree": 0.006673177083333333 }, { "tau": 3.5, "escalated_frac": 0.3914388020833333, "residual_disagree": 0.006673177083333333 }, { "tau": 3.5521356783919487, "escalated_frac": 0.4070638020833333, "residual_disagree": 0.00537109375 }, { "tau": 3.75, "escalated_frac": 0.4112955729166667, "residual_disagree": 0.005045572916666667 }, { "tau": 3.75, "escalated_frac": 0.4112955729166667, "residual_disagree": 0.005045572916666667 }, { "tau": 3.75, "escalated_frac": 0.4112955729166667, "residual_disagree": 0.005045572916666667 }, { "tau": 3.875, "escalated_frac": 0.4231770833333333, "residual_disagree": 0.004231770833333333 }, { "tau": 4.0, "escalated_frac": 0.42724609375, "residual_disagree": 0.004231770833333333 }, { "tau": 4.0, "escalated_frac": 0.42724609375, "residual_disagree": 0.004231770833333333 }, { "tau": 4.125, "escalated_frac": 0.4386393229166667, "residual_disagree": 0.0037434895833333335 }, { "tau": 4.25, "escalated_frac": 0.44287109375, "residual_disagree": 0.0037434895833333335 }, { "tau": 4.25, "escalated_frac": 0.44287109375, "residual_disagree": 0.0037434895833333335 }, { "tau": 4.375, "escalated_frac": 0.4558919270833333, "residual_disagree": 0.0029296875 }, { "tau": 4.5, "escalated_frac": 0.4591471354166667, "residual_disagree": 0.0029296875 }, { "tau": 4.5, "escalated_frac": 0.4591471354166667, "residual_disagree": 0.0029296875 }, { "tau": 4.625, "escalated_frac": 0.4698893229166667, "residual_disagree": 0.00244140625 }, { "tau": 4.75, "escalated_frac": 0.4737955729166667, "residual_disagree": 0.00244140625 }, { "tau": 4.75, "escalated_frac": 0.4737955729166667, "residual_disagree": 0.00244140625 }, { "tau": 4.875, "escalated_frac": 0.4850260416666667, "residual_disagree": 0.001953125 }, { "tau": 5.0, "escalated_frac": 0.4889322916666667, "residual_disagree": 0.001953125 }, { "tau": 5.0, "escalated_frac": 0.4889322916666667, "residual_disagree": 0.001953125 }, { "tau": 5.125, "escalated_frac": 0.49853515625, "residual_disagree": 0.001953125 }, { "tau": 5.25, "escalated_frac": 0.50390625, "residual_disagree": 0.001953125 }, { "tau": 5.25, "escalated_frac": 0.50390625, "residual_disagree": 0.001953125 }, { "tau": 5.5, "escalated_frac": 0.5149739583333334, "residual_disagree": 0.0017903645833333333 }, { "tau": 5.5, "escalated_frac": 0.5149739583333334, "residual_disagree": 0.0017903645833333333 }, { "tau": 5.625, "escalated_frac": 0.5255533854166666, "residual_disagree": 0.0016276041666666667 }, { "tau": 5.75, "escalated_frac": 0.5284830729166666, "residual_disagree": 0.0016276041666666667 }, { "tau": 5.875, "escalated_frac": 0.537109375, "residual_disagree": 0.0013020833333333333 }, { "tau": 6.0, "escalated_frac": 0.5413411458333334, "residual_disagree": 0.0013020833333333333 }, { "tau": 6.125, "escalated_frac": 0.5472005208333334, "residual_disagree": 0.0011393229166666667 }, { "tau": 6.25, "escalated_frac": 0.5491536458333334, "residual_disagree": 0.0011393229166666667 }, { "tau": 6.375, "escalated_frac": 0.55712890625, "residual_disagree": 0.0011393229166666667 }, { "tau": 6.5, "escalated_frac": 0.5597330729166666, "residual_disagree": 0.0011393229166666667 }, { "tau": 6.625, "escalated_frac": 0.5665690104166666, "residual_disagree": 0.0009765625 }, { "tau": 6.75, "escalated_frac": 0.57080078125, "residual_disagree": 0.0009765625 }, { "tau": 6.75, "escalated_frac": 0.57080078125, "residual_disagree": 0.0009765625 }, { "tau": 7.0, "escalated_frac": 0.58154296875, "residual_disagree": 0.00048828125 }, { "tau": 7.0, "escalated_frac": 0.58154296875, "residual_disagree": 0.00048828125 }, { "tau": 7.197864321608051, "escalated_frac": 0.5929361979166666, "residual_disagree": 0.0003255208333333333 }, { "tau": 7.25, "escalated_frac": 0.5929361979166666, "residual_disagree": 0.0003255208333333333 }, { "tau": 7.375, "escalated_frac": 0.6013997395833334, "residual_disagree": 0.0003255208333333333 }, { "tau": 7.5, "escalated_frac": 0.6046549479166666, "residual_disagree": 0.0003255208333333333 }, { "tau": 7.75, "escalated_frac": 0.6123046875, "residual_disagree": 0.0003255208333333333 }, { "tau": 7.875, "escalated_frac": 0.6178385416666666, "residual_disagree": 0.0003255208333333333 }, { "tau": 8.0, "escalated_frac": 0.6214192708333334, "residual_disagree": 0.0003255208333333333 }, { "tau": 8.125, "escalated_frac": 0.6272786458333334, "residual_disagree": 0.0003255208333333333 }, { "tau": 8.25, "escalated_frac": 0.6316731770833334, "residual_disagree": 0.0003255208333333333 }, { "tau": 8.375, "escalated_frac": 0.6376953125, "residual_disagree": 0.0003255208333333333 }, { "tau": 8.5, "escalated_frac": 0.640625, "residual_disagree": 0.0003255208333333333 }, { "tau": 8.625, "escalated_frac": 0.6466471354166666, "residual_disagree": 0.0003255208333333333 }, { "tau": 8.75, "escalated_frac": 0.650390625, "residual_disagree": 0.0003255208333333333 }, { "tau": 8.875, "escalated_frac": 0.6555989583333334, "residual_disagree": 0.0003255208333333333 }, { "tau": 9.0, "escalated_frac": 0.6598307291666666, "residual_disagree": 0.0003255208333333333 }, { "tau": 9.125, "escalated_frac": 0.6653645833333334, "residual_disagree": 0.0 }, { "tau": 9.25, "escalated_frac": 0.6686197916666666, "residual_disagree": 0.0 }, { "tau": 9.5, "escalated_frac": 0.6770833333333334, "residual_disagree": 0.0 }, { "tau": 9.5, "escalated_frac": 0.6770833333333334, "residual_disagree": 0.0 }, { "tau": 9.75, "escalated_frac": 0.6861979166666666, "residual_disagree": 0.0 }, { "tau": 9.9962311557789, "escalated_frac": 0.693359375, "residual_disagree": 0.0 }, { "tau": 10.125, "escalated_frac": 0.6971028645833334, "residual_disagree": 0.0 }, { "tau": 10.25, "escalated_frac": 0.6988932291666666, "residual_disagree": 0.0 }, { "tau": 10.5, "escalated_frac": 0.7068684895833334, "residual_disagree": 0.0 }, { "tau": 10.625, "escalated_frac": 0.7120768229166666, "residual_disagree": 0.0 }, { "tau": 10.75, "escalated_frac": 0.71435546875, "residual_disagree": 0.0 }, { "tau": 11.0, "escalated_frac": 0.7220052083333334, "residual_disagree": 0.0 }, { "tau": 11.125, "escalated_frac": 0.7275390625, "residual_disagree": 0.0 }, { "tau": 11.375, "escalated_frac": 0.7327473958333334, "residual_disagree": 0.0 }, { "tau": 11.5, "escalated_frac": 0.7360026041666666, "residual_disagree": 0.0 }, { "tau": 11.75, "escalated_frac": 0.7410481770833334, "residual_disagree": 0.0 }, { "tau": 12.0, "escalated_frac": 0.7478841145833334, "residual_disagree": 0.0 }, { "tau": 12.25, "escalated_frac": 0.7529296875, "residual_disagree": 0.0 }, { "tau": 12.375, "escalated_frac": 0.7566731770833334, "residual_disagree": 0.0 }, { "tau": 12.75, "escalated_frac": 0.7630208333333334, "residual_disagree": 0.0 }, { "tau": 12.875, "escalated_frac": 0.7667643229166666, "residual_disagree": 0.0 }, { "tau": 13.125, "escalated_frac": 0.7734375, "residual_disagree": 0.0 }, { "tau": 13.343592964824097, "escalated_frac": 0.77880859375, "residual_disagree": 0.0 }, { "tau": 13.5, "escalated_frac": 0.78173828125, "residual_disagree": 0.0 }, { "tau": 13.625, "escalated_frac": 0.7880859375, "residual_disagree": 0.0 }, { "tau": 13.875, "escalated_frac": 0.79345703125, "residual_disagree": 0.0 }, { "tau": 14.0, "escalated_frac": 0.7972005208333334, "residual_disagree": 0.0 }, { "tau": 14.25, "escalated_frac": 0.8025716145833334, "residual_disagree": 0.0 }, { "tau": 14.375, "escalated_frac": 0.8079427083333334, "residual_disagree": 0.0 }, { "tau": 14.5, "escalated_frac": 0.8103841145833334, "residual_disagree": 0.0 }, { "tau": 14.75, "escalated_frac": 0.8170572916666666, "residual_disagree": 0.0 }, { "tau": 14.875, "escalated_frac": 0.8212890625, "residual_disagree": 0.0 }, { "tau": 15.180276381909607, "escalated_frac": 0.8291015625, "residual_disagree": 0.0 }, { "tau": 15.5, "escalated_frac": 0.8331705729166666, "residual_disagree": 0.0 }, { "tau": 15.75, "escalated_frac": 0.8377278645833334, "residual_disagree": 0.0 }, { "tau": 16.0, "escalated_frac": 0.8435872395833334, "residual_disagree": 0.0 }, { "tau": 16.125, "escalated_frac": 0.84814453125, "residual_disagree": 0.0 }, { "tau": 16.375, "escalated_frac": 0.8528645833333334, "residual_disagree": 0.0 }, { "tau": 16.625, "escalated_frac": 0.8583984375, "residual_disagree": 0.0 }, { "tau": 16.815954773869294, "escalated_frac": 0.8642578125, "residual_disagree": 0.0 }, { "tau": 17.125, "escalated_frac": 0.8680013020833334, "residual_disagree": 0.0 }, { "tau": 17.375, "escalated_frac": 0.8738606770833334, "residual_disagree": 0.0 }, { "tau": 17.625, "escalated_frac": 0.8782552083333334, "residual_disagree": 0.0 }, { "tau": 17.75062814070361, "escalated_frac": 0.8844401041666666, "residual_disagree": 0.0 }, { "tau": 18.125, "escalated_frac": 0.8880208333333334, "residual_disagree": 0.0 }, { "tau": 18.5, "escalated_frac": 0.8937174479166666, "residual_disagree": 0.0 }, { "tau": 18.875, "escalated_frac": 0.8990885416666666, "residual_disagree": 0.0 }, { "tau": 19.125, "escalated_frac": 0.9025065104166666, "residual_disagree": 0.0 }, { "tau": 19.29396984924631, "escalated_frac": 0.9095052083333334, "residual_disagree": 0.0 }, { "tau": 19.625, "escalated_frac": 0.9138997395833334, "residual_disagree": 0.0 }, { "tau": 19.875, "escalated_frac": 0.9181315104166666, "residual_disagree": 0.0 }, { "tau": 20.25, "escalated_frac": 0.9235026041666666, "residual_disagree": 0.0 }, { "tau": 20.625, "escalated_frac": 0.9285481770833334, "residual_disagree": 0.0 }, { "tau": 21.0, "escalated_frac": 0.93310546875, "residual_disagree": 0.0 }, { "tau": 21.375, "escalated_frac": 0.9383138020833334, "residual_disagree": 0.0 }, { "tau": 21.875, "escalated_frac": 0.9435221354166666, "residual_disagree": 0.0 }, { "tau": 22.25, "escalated_frac": 0.9495442708333334, "residual_disagree": 0.0 }, { "tau": 22.625, "escalated_frac": 0.9542643229166666, "residual_disagree": 0.0 }, { "tau": 23.130653266331706, "escalated_frac": 0.9597981770833334, "residual_disagree": 0.0 }, { "tau": 23.75, "escalated_frac": 0.9645182291666666, "residual_disagree": 0.0 }, { "tau": 24.375, "escalated_frac": 0.9689127604166666, "residual_disagree": 0.0 }, { "tau": 25.206658291457302, "escalated_frac": 0.9747721354166666, "residual_disagree": 0.0 }, { "tau": 25.875, "escalated_frac": 0.97900390625, "residual_disagree": 0.0 }, { "tau": 26.875, "escalated_frac": 0.9845377604166666, "residual_disagree": 0.0 }, { "tau": 28.125, "escalated_frac": 0.9894205729166666, "residual_disagree": 0.0 }, { "tau": 29.766331658291392, "escalated_frac": 0.9949544270833334, "residual_disagree": 0.0 }, { "tau": 35.25, "escalated_frac": 0.9998372395833334, "residual_disagree": 0.0 } ], "per_domain": { "code": { "agreement_rate": 0.857421875, "auroc": 0.8844041563952945 }, "math": { "agreement_rate": 0.9365234375, "auroc": 0.9211117349803482 }, "chat": { "agreement_rate": 0.8271484375, "auroc": 0.880055229824105 }, "french": { "agreement_rate": 0.87109375, "auroc": 0.9024663677130045 }, "english": { "agreement_rate": 0.84765625, "auroc": 0.9031445704832801 }, "reasoning": { "agreement_rate": 0.904296875, "auroc": 0.904030942830696 } }, "escalation_frac_for_99pct": 0.3658854166666667 }, "8": { "bits": 8, "n_positions": 6144, "agreement_rate": 0.9803059895833334, "mean_kl_ref_q": 0.003948633559048176, "auroc_margin_predicts_disagreement": 0.9745905159697743, "median_margin_agree": 6.25, "median_margin_disagree": 0.25, "escalation_curve": [ { "tau": 0.0, "escalated_frac": 0.0, "residual_disagree": 0.019694010416666668 }, { "tau": 0.0, "escalated_frac": 0.0, "residual_disagree": 0.019694010416666668 }, { "tau": 0.0, "escalated_frac": 0.0, "residual_disagree": 0.019694010416666668 }, { "tau": 0.125, "escalated_frac": 0.012858072916666666, "residual_disagree": 0.014973958333333334 }, { "tau": 0.125, "escalated_frac": 0.012858072916666666, "residual_disagree": 0.014973958333333334 }, { "tau": 0.25, "escalated_frac": 0.022623697916666668, "residual_disagree": 0.011393229166666666 }, { "tau": 0.25, "escalated_frac": 0.022623697916666668, "residual_disagree": 0.011393229166666666 }, { "tau": 0.25, "escalated_frac": 0.022623697916666668, "residual_disagree": 0.011393229166666666 }, { "tau": 0.25, "escalated_frac": 0.022623697916666668, "residual_disagree": 0.011393229166666666 }, { "tau": 0.25, "escalated_frac": 0.022623697916666668, "residual_disagree": 0.011393229166666666 }, { "tau": 0.25, "escalated_frac": 0.022623697916666668, "residual_disagree": 0.011393229166666666 }, { "tau": 0.375, "escalated_frac": 0.051106770833333336, "residual_disagree": 0.0037434895833333335 }, { "tau": 0.5, "escalated_frac": 0.059895833333333336, "residual_disagree": 0.0027669270833333335 }, { "tau": 0.5, "escalated_frac": 0.059895833333333336, "residual_disagree": 0.0027669270833333335 }, { "tau": 0.5, "escalated_frac": 0.059895833333333336, "residual_disagree": 0.0027669270833333335 }, { "tau": 0.5, "escalated_frac": 0.059895833333333336, "residual_disagree": 0.0027669270833333335 }, { "tau": 0.5, "escalated_frac": 0.059895833333333336, "residual_disagree": 0.0027669270833333335 }, { "tau": 0.625, "escalated_frac": 0.08154296875, "residual_disagree": 0.0011393229166666667 }, { "tau": 0.75, "escalated_frac": 0.08935546875, "residual_disagree": 0.0011393229166666667 }, { "tau": 0.75, "escalated_frac": 0.08935546875, "residual_disagree": 0.0011393229166666667 }, { "tau": 0.75, "escalated_frac": 0.08935546875, "residual_disagree": 0.0011393229166666667 }, { "tau": 0.75, "escalated_frac": 0.08935546875, "residual_disagree": 0.0011393229166666667 }, { "tau": 0.75, "escalated_frac": 0.08935546875, "residual_disagree": 0.0011393229166666667 }, { "tau": 0.875, "escalated_frac": 0.1142578125, "residual_disagree": 0.00048828125 }, { "tau": 0.875, "escalated_frac": 0.1142578125, "residual_disagree": 0.00048828125 }, { "tau": 1.0, "escalated_frac": 0.12109375, "residual_disagree": 0.00048828125 }, { "tau": 1.0, "escalated_frac": 0.12109375, "residual_disagree": 0.00048828125 }, { "tau": 1.0, "escalated_frac": 0.12109375, "residual_disagree": 0.00048828125 }, { "tau": 1.0, "escalated_frac": 0.12109375, "residual_disagree": 0.00048828125 }, { "tau": 1.125, "escalated_frac": 0.14111328125, "residual_disagree": 0.0003255208333333333 }, { "tau": 1.25, "escalated_frac": 0.14713541666666666, "residual_disagree": 0.0003255208333333333 }, { "tau": 1.25, "escalated_frac": 0.14713541666666666, "residual_disagree": 0.0003255208333333333 }, { "tau": 1.25, "escalated_frac": 0.14713541666666666, "residual_disagree": 0.0003255208333333333 }, { "tau": 1.25, "escalated_frac": 0.14713541666666666, "residual_disagree": 0.0003255208333333333 }, { "tau": 1.375, "escalated_frac": 0.17041015625, "residual_disagree": 0.0 }, { "tau": 1.5, "escalated_frac": 0.1748046875, "residual_disagree": 0.0 }, { "tau": 1.5, "escalated_frac": 0.1748046875, "residual_disagree": 0.0 }, { "tau": 1.5, "escalated_frac": 0.1748046875, "residual_disagree": 0.0 }, { "tau": 1.5, "escalated_frac": 0.1748046875, "residual_disagree": 0.0 }, { "tau": 1.625, "escalated_frac": 0.19547526041666666, "residual_disagree": 0.0 }, { "tau": 1.75, "escalated_frac": 0.20084635416666666, "residual_disagree": 0.0 }, { "tau": 1.75, "escalated_frac": 0.20084635416666666, "residual_disagree": 0.0 }, { "tau": 1.75, "escalated_frac": 0.20084635416666666, "residual_disagree": 0.0 }, { "tau": 1.75, "escalated_frac": 0.20084635416666666, "residual_disagree": 0.0 }, { "tau": 1.75, "escalated_frac": 0.20084635416666666, "residual_disagree": 0.0 }, { "tau": 1.875, "escalated_frac": 0.22233072916666666, "residual_disagree": 0.0 }, { "tau": 2.0, "escalated_frac": 0.22672526041666666, "residual_disagree": 0.0 }, { "tau": 2.0, "escalated_frac": 0.22672526041666666, "residual_disagree": 0.0 }, { "tau": 2.0, "escalated_frac": 0.22672526041666666, "residual_disagree": 0.0 }, { "tau": 2.125, "escalated_frac": 0.2431640625, "residual_disagree": 0.0 }, { "tau": 2.25, "escalated_frac": 0.24755859375, "residual_disagree": 0.0 }, { "tau": 2.25, "escalated_frac": 0.24755859375, "residual_disagree": 0.0 }, { "tau": 2.25, "escalated_frac": 0.24755859375, "residual_disagree": 0.0 }, { "tau": 2.375, "escalated_frac": 0.2644856770833333, "residual_disagree": 0.0 }, { "tau": 2.5, "escalated_frac": 0.2696940104166667, "residual_disagree": 0.0 }, { "tau": 2.5, "escalated_frac": 0.2696940104166667, "residual_disagree": 0.0 }, { "tau": 2.5, "escalated_frac": 0.2696940104166667, "residual_disagree": 0.0 }, { "tau": 2.5, "escalated_frac": 0.2696940104166667, "residual_disagree": 0.0 }, { "tau": 2.625, "escalated_frac": 0.2872721354166667, "residual_disagree": 0.0 }, { "tau": 2.75, "escalated_frac": 0.2931315104166667, "residual_disagree": 0.0 }, { "tau": 2.75, "escalated_frac": 0.2931315104166667, "residual_disagree": 0.0 }, { "tau": 2.75, "escalated_frac": 0.2931315104166667, "residual_disagree": 0.0 }, { "tau": 2.875, "escalated_frac": 0.3089192708333333, "residual_disagree": 0.0 }, { "tau": 3.0, "escalated_frac": 0.3136393229166667, "residual_disagree": 0.0 }, { "tau": 3.0, "escalated_frac": 0.3136393229166667, "residual_disagree": 0.0 }, { "tau": 3.0, "escalated_frac": 0.3136393229166667, "residual_disagree": 0.0 }, { "tau": 3.125, "escalated_frac": 0.3304036458333333, "residual_disagree": 0.0 }, { "tau": 3.25, "escalated_frac": 0.33447265625, "residual_disagree": 0.0 }, { "tau": 3.25, "escalated_frac": 0.33447265625, "residual_disagree": 0.0 }, { "tau": 3.25, "escalated_frac": 0.33447265625, "residual_disagree": 0.0 }, { "tau": 3.375, "escalated_frac": 0.3504231770833333, "residual_disagree": 0.0 }, { "tau": 3.5, "escalated_frac": 0.35400390625, "residual_disagree": 0.0 }, { "tau": 3.5, "escalated_frac": 0.35400390625, "residual_disagree": 0.0 }, { "tau": 3.5, "escalated_frac": 0.35400390625, "residual_disagree": 0.0 }, { "tau": 3.75, "escalated_frac": 0.37158203125, "residual_disagree": 0.0 }, { "tau": 3.75, "escalated_frac": 0.37158203125, "residual_disagree": 0.0 }, { "tau": 3.75, "escalated_frac": 0.37158203125, "residual_disagree": 0.0 }, { "tau": 3.75, "escalated_frac": 0.37158203125, "residual_disagree": 0.0 }, { "tau": 3.875, "escalated_frac": 0.3888346354166667, "residual_disagree": 0.0 }, { "tau": 4.0, "escalated_frac": 0.3929036458333333, "residual_disagree": 0.0 }, { "tau": 4.0, "escalated_frac": 0.3929036458333333, "residual_disagree": 0.0 }, { "tau": 4.177135678391949, "escalated_frac": 0.4070638020833333, "residual_disagree": 0.0 }, { "tau": 4.25, "escalated_frac": 0.4070638020833333, "residual_disagree": 0.0 }, { "tau": 4.25, "escalated_frac": 0.4070638020833333, "residual_disagree": 0.0 }, { "tau": 4.375, "escalated_frac": 0.42138671875, "residual_disagree": 0.0 }, { "tau": 4.5, "escalated_frac": 0.42529296875, "residual_disagree": 0.0 }, { "tau": 4.5, "escalated_frac": 0.42529296875, "residual_disagree": 0.0 }, { "tau": 4.625, "escalated_frac": 0.4358723958333333, "residual_disagree": 0.0 }, { "tau": 4.75, "escalated_frac": 0.43896484375, "residual_disagree": 0.0 }, { "tau": 4.75, "escalated_frac": 0.43896484375, "residual_disagree": 0.0 }, { "tau": 4.875, "escalated_frac": 0.4503580729166667, "residual_disagree": 0.0 }, { "tau": 5.0, "escalated_frac": 0.4529622395833333, "residual_disagree": 0.0 }, { "tau": 5.0, "escalated_frac": 0.4529622395833333, "residual_disagree": 0.0 }, { "tau": 5.25, "escalated_frac": 0.4658203125, "residual_disagree": 0.0 }, { "tau": 5.25, "escalated_frac": 0.4658203125, "residual_disagree": 0.0 }, { "tau": 5.375, "escalated_frac": 0.4752604166666667, "residual_disagree": 0.0 }, { "tau": 5.5, "escalated_frac": 0.4775390625, "residual_disagree": 0.0 }, { "tau": 5.625, "escalated_frac": 0.4871419270833333, "residual_disagree": 0.0 }, { "tau": 5.75, "escalated_frac": 0.4900716145833333, "residual_disagree": 0.0 }, { "tau": 5.875, "escalated_frac": 0.4972330729166667, "residual_disagree": 0.0 }, { "tau": 6.0, "escalated_frac": 0.49951171875, "residual_disagree": 0.0 }, { "tau": 6.125, "escalated_frac": 0.5066731770833334, "residual_disagree": 0.0 }, { "tau": 6.25, "escalated_frac": 0.50927734375, "residual_disagree": 0.0 }, { "tau": 6.25, "escalated_frac": 0.50927734375, "residual_disagree": 0.0 }, { "tau": 6.5, "escalated_frac": 0.52099609375, "residual_disagree": 0.0 }, { "tau": 6.5, "escalated_frac": 0.52099609375, "residual_disagree": 0.0 }, { "tau": 6.75, "escalated_frac": 0.53173828125, "residual_disagree": 0.0 }, { "tau": 6.75, "escalated_frac": 0.53173828125, "residual_disagree": 0.0 }, { "tau": 6.875, "escalated_frac": 0.541015625, "residual_disagree": 0.0 }, { "tau": 7.0, "escalated_frac": 0.54296875, "residual_disagree": 0.0 }, { "tau": 7.125, "escalated_frac": 0.5499674479166666, "residual_disagree": 0.0 }, { "tau": 7.25, "escalated_frac": 0.5540364583333334, "residual_disagree": 0.0 }, { "tau": 7.5, "escalated_frac": 0.5626627604166666, "residual_disagree": 0.0 }, { "tau": 7.5, "escalated_frac": 0.5626627604166666, "residual_disagree": 0.0 }, { "tau": 7.75, "escalated_frac": 0.5709635416666666, "residual_disagree": 0.0 }, { "tau": 7.75, "escalated_frac": 0.5709635416666666, "residual_disagree": 0.0 }, { "tau": 8.0, "escalated_frac": 0.5802408854166666, "residual_disagree": 0.0 }, { "tau": 8.125, "escalated_frac": 0.5852864583333334, "residual_disagree": 0.0 }, { "tau": 8.25, "escalated_frac": 0.5880533854166666, "residual_disagree": 0.0 }, { "tau": 8.5, "escalated_frac": 0.59765625, "residual_disagree": 0.0 }, { "tau": 8.5, "escalated_frac": 0.59765625, "residual_disagree": 0.0 }, { "tau": 8.648869346733647, "escalated_frac": 0.6080729166666666, "residual_disagree": 0.0 }, { "tau": 8.75, "escalated_frac": 0.6080729166666666, "residual_disagree": 0.0 }, { "tau": 9.0, "escalated_frac": 0.6160481770833334, "residual_disagree": 0.0 }, { "tau": 9.125, "escalated_frac": 0.6222330729166666, "residual_disagree": 0.0 }, { "tau": 9.25, "escalated_frac": 0.6253255208333334, "residual_disagree": 0.0 }, { "tau": 9.375, "escalated_frac": 0.6326497395833334, "residual_disagree": 0.0 }, { "tau": 9.5, "escalated_frac": 0.63623046875, "residual_disagree": 0.0 }, { "tau": 9.625, "escalated_frac": 0.6416015625, "residual_disagree": 0.0 }, { "tau": 9.75, "escalated_frac": 0.6433919270833334, "residual_disagree": 0.0 }, { "tau": 10.0, "escalated_frac": 0.6531575520833334, "residual_disagree": 0.0 }, { "tau": 10.0, "escalated_frac": 0.6531575520833334, "residual_disagree": 0.0 }, { "tau": 10.25, "escalated_frac": 0.66064453125, "residual_disagree": 0.0 }, { "tau": 10.5, "escalated_frac": 0.6673177083333334, "residual_disagree": 0.0 }, { "tau": 10.75, "escalated_frac": 0.6726888020833334, "residual_disagree": 0.0 }, { "tau": 10.75, "escalated_frac": 0.6726888020833334, "residual_disagree": 0.0 }, { "tau": 11.0, "escalated_frac": 0.681640625, "residual_disagree": 0.0 }, { "tau": 11.25, "escalated_frac": 0.6883138020833334, "residual_disagree": 0.0 }, { "tau": 11.3712311557789, "escalated_frac": 0.693359375, "residual_disagree": 0.0 }, { "tau": 11.5, "escalated_frac": 0.6954752604166666, "residual_disagree": 0.0 }, { "tau": 11.75, "escalated_frac": 0.7021484375, "residual_disagree": 0.0 }, { "tau": 11.875, "escalated_frac": 0.70556640625, "residual_disagree": 0.0 }, { "tau": 12.0, "escalated_frac": 0.708984375, "residual_disagree": 0.0 }, { "tau": 12.25, "escalated_frac": 0.71728515625, "residual_disagree": 0.0 }, { "tau": 12.375, "escalated_frac": 0.7205403645833334, "residual_disagree": 0.0 }, { "tau": 12.625, "escalated_frac": 0.7278645833333334, "residual_disagree": 0.0 }, { "tau": 12.75, "escalated_frac": 0.7301432291666666, "residual_disagree": 0.0 }, { "tau": 13.0, "escalated_frac": 0.7371419270833334, "residual_disagree": 0.0 }, { "tau": 13.25, "escalated_frac": 0.7425130208333334, "residual_disagree": 0.0 }, { "tau": 13.5, "escalated_frac": 0.7478841145833334, "residual_disagree": 0.0 }, { "tau": 13.75, "escalated_frac": 0.7527669270833334, "residual_disagree": 0.0 }, { "tau": 14.0, "escalated_frac": 0.7584635416666666, "residual_disagree": 0.0 }, { "tau": 14.125, "escalated_frac": 0.7633463541666666, "residual_disagree": 0.0 }, { "tau": 14.25, "escalated_frac": 0.76611328125, "residual_disagree": 0.0 }, { "tau": 14.5, "escalated_frac": 0.7716471354166666, "residual_disagree": 0.0 }, { "tau": 14.75, "escalated_frac": 0.7786458333333334, "residual_disagree": 0.0 }, { "tau": 14.875, "escalated_frac": 0.7830403645833334, "residual_disagree": 0.0 }, { "tau": 15.0, "escalated_frac": 0.7867838541666666, "residual_disagree": 0.0 }, { "tau": 15.25, "escalated_frac": 0.7937825520833334, "residual_disagree": 0.0 }, { "tau": 15.375, "escalated_frac": 0.796875, "residual_disagree": 0.0 }, { "tau": 15.5, "escalated_frac": 0.8009440104166666, "residual_disagree": 0.0 }, { "tau": 15.75, "escalated_frac": 0.8082682291666666, "residual_disagree": 0.0 }, { "tau": 15.875, "escalated_frac": 0.8136393229166666, "residual_disagree": 0.0 }, { "tau": 16.087939698492505, "escalated_frac": 0.8190104166666666, "residual_disagree": 0.0 }, { "tau": 16.375, "escalated_frac": 0.8230794270833334, "residual_disagree": 0.0 }, { "tau": 16.625, "escalated_frac": 0.828125, "residual_disagree": 0.0 }, { "tau": 16.75, "escalated_frac": 0.8312174479166666, "residual_disagree": 0.0 }, { "tau": 17.0, "escalated_frac": 0.8380533854166666, "residual_disagree": 0.0 }, { "tau": 17.25, "escalated_frac": 0.8435872395833334, "residual_disagree": 0.0 }, { "tau": 17.5, "escalated_frac": 0.8479817708333334, "residual_disagree": 0.0 }, { "tau": 17.75, "escalated_frac": 0.8528645833333334, "residual_disagree": 0.0 }, { "tau": 18.0, "escalated_frac": 0.8575846354166666, "residual_disagree": 0.0 }, { "tau": 18.190954773869294, "escalated_frac": 0.8642578125, "residual_disagree": 0.0 }, { "tau": 18.5, "escalated_frac": 0.86767578125, "residual_disagree": 0.0 }, { "tau": 18.75, "escalated_frac": 0.8733723958333334, "residual_disagree": 0.0 }, { "tau": 19.125, "escalated_frac": 0.8790690104166666, "residual_disagree": 0.0 }, { "tau": 19.5, "escalated_frac": 0.88427734375, "residual_disagree": 0.0 }, { "tau": 19.75, "escalated_frac": 0.8885091145833334, "residual_disagree": 0.0 }, { "tau": 20.125, "escalated_frac": 0.8935546875, "residual_disagree": 0.0 }, { "tau": 20.5, "escalated_frac": 0.8982747395833334, "residual_disagree": 0.0 }, { "tau": 20.8103015075377, "escalated_frac": 0.9044596354166666, "residual_disagree": 0.0 }, { "tau": 21.125, "escalated_frac": 0.9090169270833334, "residual_disagree": 0.0 }, { "tau": 21.5, "escalated_frac": 0.91357421875, "residual_disagree": 0.0 }, { "tau": 21.875, "escalated_frac": 0.9182942708333334, "residual_disagree": 0.0 }, { "tau": 22.25, "escalated_frac": 0.9225260416666666, "residual_disagree": 0.0 }, { "tau": 22.625, "escalated_frac": 0.9285481770833334, "residual_disagree": 0.0 }, { "tau": 23.0, "escalated_frac": 0.93310546875, "residual_disagree": 0.0 }, { "tau": 23.5, "escalated_frac": 0.9391276041666666, "residual_disagree": 0.0 }, { "tau": 23.875, "escalated_frac": 0.9436848958333334, "residual_disagree": 0.0 }, { "tau": 24.375, "escalated_frac": 0.9495442708333334, "residual_disagree": 0.0 }, { "tau": 24.88599246231155, "escalated_frac": 0.9547526041666666, "residual_disagree": 0.0 }, { "tau": 25.25, "escalated_frac": 0.9578450520833334, "residual_disagree": 0.0 }, { "tau": 25.875, "escalated_frac": 0.9645182291666666, "residual_disagree": 0.0 }, { "tau": 26.695979899497388, "escalated_frac": 0.9697265625, "residual_disagree": 0.0 }, { "tau": 27.25, "escalated_frac": 0.9744466145833334, "residual_disagree": 0.0 }, { "tau": 28.25, "escalated_frac": 0.9794921875, "residual_disagree": 0.0 }, { "tau": 29.5, "escalated_frac": 0.9845377604166666, "residual_disagree": 0.0 }, { "tau": 30.5, "escalated_frac": 0.98974609375, "residual_disagree": 0.0 }, { "tau": 32.032663316582784, "escalated_frac": 0.9949544270833334, "residual_disagree": 0.0 }, { "tau": 37.5, "escalated_frac": 0.9998372395833334, "residual_disagree": 0.0 } ], "per_domain": { "code": { "agreement_rate": 0.982421875, "auroc": 0.9611221559531699 }, "math": { "agreement_rate": 0.9912109375, "auroc": 0.9871373836891079 }, "chat": { "agreement_rate": 0.974609375, "auroc": 0.968937875751503 }, "french": { "agreement_rate": 0.978515625, "auroc": 0.981854472872437 }, "english": { "agreement_rate": 0.9677734375, "auroc": 0.9724490107941167 }, "reasoning": { "agreement_rate": 0.9873046875, "auroc": 0.9800654340713688 } }, "escalation_frac_for_99pct": 0.051106770833333336 } } }