1 arm n_runs n_tasks acceptance_taskw acceptance_minw det_completion critical_error_rate quality_mean quality_sd timeout_rate cost_mean cost_sd cost_per_accepted wall_median_s tool_calls_mean tool_errors_mean app_transitions_mean abstention_rate20 A0 30 10 0.000 0.000 0.000 0.200 27.389 17.403 0.0 0.710 0.416 NaN 161.95 29.433 11.867 0.000 0.90031 A1 30 10 0.867 0.885 0.967 0.033 82.347 18.500 0.0 0.823 0.230 0.950 217.65 12.867 0.900 3.267 0.20042 A2 30 10 0.900 0.909 0.967 0.000 84.972 9.977 0.0 0.783 0.190 0.870 213.85 12.367 0.667 4.433 0.06753 A3 30 10 0.900 0.926 1.000 0.000 85.694 11.378 0.0 0.906 0.259 1.006 218.80 17.200 0.867 9.300 0.13364 A4 30 10 0.833 0.844 0.967 0.000 84.792 10.587 0.0 0.926 0.221 1.111 230.15 18.800 2.000 9.267 0.1007 contrast kind outcome estimate ci_lo ci_hi perm_p J estimate_minutes_weighted holm_p80 A3-A1 primary S 0.033 0.000 0.100 1.000 10 0.041 NaN91 A3-A1 primary D 0.033 0.000 0.100 1.000 10 0.041 NaN102 A3-A1 primary E -0.033 -0.100 0.000 1.000 10 -0.041 NaN113 A3-A1 primary quality 3.347 -0.806 9.556 0.300 10 4.571 NaN124 A1-A0 secondary S 0.867 0.667 1.000 0.004 10 0.885 0.012135 A1-A0 secondary D 0.967 0.900 1.000 0.002 10 0.959 NaN146 A1-A0 secondary E -0.167 -0.467 0.067 0.499 10 -0.144 NaN157 A1-A0 secondary quality 54.958 40.444 68.236 0.002 10 57.659 NaN168 A4-A3 secondary S -0.067 -0.167 0.000 0.499 10 -0.082 0.998179 A4-A3 secondary D -0.033 -0.100 0.000 1.000 10 -0.041 NaN1810 A4-A3 secondary E 0.000 0.000 0.000 1.000 10 0.000 NaN1911 A4-A3 secondary quality -0.903 -2.709 0.973 0.395 10 -1.392 NaN2012 A2-A1 secondary S 0.033 -0.067 0.133 1.000 10 0.025 1.0002113 A2-A1 secondary D 0.000 -0.100 0.100 1.000 10 0.000 NaN2214 A2-A1 secondary E -0.033 -0.100 0.000 1.000 10 -0.041 NaN2315 A2-A1 secondary quality 2.625 -1.750 8.792 0.534 10 2.965 NaN24{25 "n_A4": 30,26 "n_failed_first_selfcheck": 0,27 "n_recovery_attempted": 0,28 "n_recovered": 0,29 "ARI": null,30 "acceptance_given_recovery": null,31 "mean_recovery_cost": 0.032}33{34 "krippendorff_alpha_interval": 0.9730624740153422,35 "pairwise_pearson": {36 "0": {37 "0": 1.0,38 "1": 0.964,39 "2": 0.96640 },41 "1": {42 "0": 0.964,43 "1": 1.0,44 "2": 0.9945 },46 "2": {47 "0": 0.966,48 "1": 0.99,49 "2": 1.050 }51 },52 "n_runs": 15053}54