arm n_runs n_tasks acceptance_taskw acceptance_minw det_completion critical_error_rate quality_mean quality_sd timeout_rate cost_mean cost_sd cost_per_accepted wall_median_s tool_calls_mean tool_errors_mean app_transitions_mean abstention_rate 0 A0 30 10 0.000 0.000 0.000 0.200 27.389 17.403 0.0 0.710 0.416 NaN 161.95 29.433 11.867 0.000 0.900 1 A1 30 10 0.867 0.885 0.967 0.033 82.347 18.500 0.0 0.823 0.230 0.950 217.65 12.867 0.900 3.267 0.200 2 A2 30 10 0.900 0.909 0.967 0.000 84.972 9.977 0.0 0.783 0.190 0.870 213.85 12.367 0.667 4.433 0.067 3 A3 30 10 0.900 0.926 1.000 0.000 85.694 11.378 0.0 0.906 0.259 1.006 218.80 17.200 0.867 9.300 0.133 4 A4 30 10 0.833 0.844 0.967 0.000 84.792 10.587 0.0 0.926 0.221 1.111 230.15 18.800 2.000 9.267 0.100 contrast kind outcome estimate ci_lo ci_hi perm_p J estimate_minutes_weighted holm_p 0 A3-A1 primary S 0.033 0.000 0.100 1.000 10 0.041 NaN 1 A3-A1 primary D 0.033 0.000 0.100 1.000 10 0.041 NaN 2 A3-A1 primary E -0.033 -0.100 0.000 1.000 10 -0.041 NaN 3 A3-A1 primary quality 3.347 -0.806 9.556 0.300 10 4.571 NaN 4 A1-A0 secondary S 0.867 0.667 1.000 0.004 10 0.885 0.012 5 A1-A0 secondary D 0.967 0.900 1.000 0.002 10 0.959 NaN 6 A1-A0 secondary E -0.167 -0.467 0.067 0.499 10 -0.144 NaN 7 A1-A0 secondary quality 54.958 40.444 68.236 0.002 10 57.659 NaN 8 A4-A3 secondary S -0.067 -0.167 0.000 0.499 10 -0.082 0.998 9 A4-A3 secondary D -0.033 -0.100 0.000 1.000 10 -0.041 NaN 10 A4-A3 secondary E 0.000 0.000 0.000 1.000 10 0.000 NaN 11 A4-A3 secondary quality -0.903 -2.709 0.973 0.395 10 -1.392 NaN 12 A2-A1 secondary S 0.033 -0.067 0.133 1.000 10 0.025 1.000 13 A2-A1 secondary D 0.000 -0.100 0.100 1.000 10 0.000 NaN 14 A2-A1 secondary E -0.033 -0.100 0.000 1.000 10 -0.041 NaN 15 A2-A1 secondary quality 2.625 -1.750 8.792 0.534 10 2.965 NaN { "n_A4": 30, "n_failed_first_selfcheck": 0, "n_recovery_attempted": 0, "n_recovered": 0, "ARI": null, "acceptance_given_recovery": null, "mean_recovery_cost": 0.0 } { "krippendorff_alpha_interval": 0.9730624740153422, "pairwise_pearson": { "0": { "0": 1.0, "1": 0.964, "2": 0.966 }, "1": { "0": 0.964, "1": 1.0, "2": 0.99 }, "2": { "0": 0.966, "1": 0.99, "2": 1.0 } }, "n_runs": 150 }