WP42: pilot (150 runs), model sensitivities, recovery experiment, analysis tables/figures, phase 5 audit
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
97 changed files +4,158 −19
added
analysis/derived/estimates_pilot.json
+404 −0
@@ -0,0 +1,404 @@ | ||
| 1 | +{ | |
| 2 | + "tag": "pilot", | |
| 3 | + "q": 70, | |
| 4 | + "n_runs": 150, | |
| 5 | + "n_tasks": 10, | |
| 6 | + "total_cost_usd": 124.42590039999999, | |
| 7 | + "by_arm": [ | |
| 8 | + { | |
| 9 | + "arm": "A0", | |
| 10 | + "n_runs": 30, | |
| 11 | + "n_tasks": 10, | |
| 12 | + "acceptance_taskw": 0.0, | |
| 13 | + "acceptance_minw": 0.0, | |
| 14 | + "det_completion": 0.0, | |
| 15 | + "critical_error_rate": 0.2, | |
| 16 | + "quality_mean": 27.38888888888889, | |
| 17 | + "quality_sd": 17.40324007207812, | |
| 18 | + "timeout_rate": 0.0, | |
| 19 | + "cost_mean": 0.7095971266666666, | |
| 20 | + "cost_sd": 0.41578295381532226, | |
| 21 | + "cost_per_accepted": NaN, | |
| 22 | + "wall_median_s": 161.95, | |
| 23 | + "tool_calls_mean": 29.433333333333334, | |
| 24 | + "tool_errors_mean": 11.866666666666667, | |
| 25 | + "app_transitions_mean": 0.0, | |
| 26 | + "abstention_rate": 0.9 | |
| 27 | + }, | |
| 28 | + { | |
| 29 | + "arm": "A1", | |
| 30 | + "n_runs": 30, | |
| 31 | + "n_tasks": 10, | |
| 32 | + "acceptance_taskw": 0.8666666666666666, | |
| 33 | + "acceptance_minw": 0.8847736625514403, | |
| 34 | + "det_completion": 0.9666666666666666, | |
| 35 | + "critical_error_rate": 0.03333333333333333, | |
| 36 | + "quality_mean": 82.34722222222221, | |
| 37 | + "quality_sd": 18.50024701260657, | |
| 38 | + "timeout_rate": 0.0, | |
| 39 | + "cost_mean": 0.82316396, | |
| 40 | + "cost_sd": 0.22962883806934387, | |
| 41 | + "cost_per_accepted": 0.9498045692307693, | |
| 42 | + "wall_median_s": 217.65, | |
| 43 | + "tool_calls_mean": 12.866666666666667, | |
| 44 | + "tool_errors_mean": 0.9, | |
| 45 | + "app_transitions_mean": 3.2666666666666666, | |
| 46 | + "abstention_rate": 0.2 | |
| 47 | + }, | |
| 48 | + { | |
| 49 | + "arm": "A2", | |
| 50 | + "n_runs": 30, | |
| 51 | + "n_tasks": 10, | |
| 52 | + "acceptance_taskw": 0.9, | |
| 53 | + "acceptance_minw": 0.9094650205761317, | |
| 54 | + "det_completion": 0.9666666666666666, | |
| 55 | + "critical_error_rate": 0.0, | |
| 56 | + "quality_mean": 84.97222222222221, | |
| 57 | + "quality_sd": 9.97718502109102, | |
| 58 | + "timeout_rate": 0.0, | |
| 59 | + "cost_mean": 0.78344624, | |
| 60 | + "cost_sd": 0.18990073897126578, | |
| 61 | + "cost_per_accepted": 0.8704958222222222, | |
| 62 | + "wall_median_s": 213.85000000000002, | |
| 63 | + "tool_calls_mean": 12.366666666666667, | |
| 64 | + "tool_errors_mean": 0.6666666666666666, | |
| 65 | + "app_transitions_mean": 4.433333333333334, | |
| 66 | + "abstention_rate": 0.06666666666666667 | |
| 67 | + }, | |
| 68 | + { | |
| 69 | + "arm": "A3", | |
| 70 | + "n_runs": 30, | |
| 71 | + "n_tasks": 10, | |
| 72 | + "acceptance_taskw": 0.9, | |
| 73 | + "acceptance_minw": 0.9259259259259259, | |
| 74 | + "det_completion": 1.0, | |
| 75 | + "critical_error_rate": 0.0, | |
| 76 | + "quality_mean": 85.69444444444444, | |
| 77 | + "quality_sd": 11.378487914859686, | |
| 78 | + "timeout_rate": 0.0, | |
| 79 | + "cost_mean": 0.9055274533333333, | |
| 80 | + "cost_sd": 0.258626810400623, | |
| 81 | + "cost_per_accepted": 1.0061416148148148, | |
| 82 | + "wall_median_s": 218.8, | |
| 83 | + "tool_calls_mean": 17.2, | |
| 84 | + "tool_errors_mean": 0.8666666666666667, | |
| 85 | + "app_transitions_mean": 9.3, | |
| 86 | + "abstention_rate": 0.13333333333333333 | |
| 87 | + }, | |
| 88 | + { | |
| 89 | + "arm": "A4", | |
| 90 | + "n_runs": 30, | |
| 91 | + "n_tasks": 10, | |
| 92 | + "acceptance_taskw": 0.8333333333333333, | |
| 93 | + "acceptance_minw": 0.8436213991769548, | |
| 94 | + "det_completion": 0.9666666666666666, | |
| 95 | + "critical_error_rate": 0.0, | |
| 96 | + "quality_mean": 84.79166666666667, | |
| 97 | + "quality_sd": 10.586687177880476, | |
| 98 | + "timeout_rate": 0.0, | |
| 99 | + "cost_mean": 0.9257952333333334, | |
| 100 | + "cost_sd": 0.22072943537881048, | |
| 101 | + "cost_per_accepted": 1.11095428, | |
| 102 | + "wall_median_s": 230.15, | |
| 103 | + "tool_calls_mean": 18.8, | |
| 104 | + "tool_errors_mean": 2.0, | |
| 105 | + "app_transitions_mean": 9.266666666666667, | |
| 106 | + "abstention_rate": 0.1 | |
| 107 | + } | |
| 108 | + ], | |
| 109 | + "contrasts": [ | |
| 110 | + { | |
| 111 | + "contrast": "A3-A1", | |
| 112 | + "kind": "primary", | |
| 113 | + "outcome": "S", | |
| 114 | + "estimate": 0.03333333333333334, | |
| 115 | + "ci_lo": 0.0, | |
| 116 | + "ci_hi": 0.1, | |
| 117 | + "perm_p": 1.0, | |
| 118 | + "J": 10, | |
| 119 | + "estimate_minutes_weighted": 0.041152263374485604, | |
| 120 | + "holm_p": NaN | |
| 121 | + }, | |
| 122 | + { | |
| 123 | + "contrast": "A3-A1", | |
| 124 | + "kind": "primary", | |
| 125 | + "outcome": "D", | |
| 126 | + "estimate": 0.03333333333333334, | |
| 127 | + "ci_lo": 0.0, | |
| 128 | + "ci_hi": 0.1, | |
| 129 | + "perm_p": 1.0, | |
| 130 | + "J": 10, | |
| 131 | + "estimate_minutes_weighted": 0.041152263374485604, | |
| 132 | + "holm_p": NaN | |
| 133 | + }, | |
| 134 | + { | |
| 135 | + "contrast": "A3-A1", | |
| 136 | + "kind": "primary", | |
| 137 | + "outcome": "E", | |
| 138 | + "estimate": -0.03333333333333333, | |
| 139 | + "ci_lo": -0.1, | |
| 140 | + "ci_hi": 0.0, | |
| 141 | + "perm_p": 1.0, | |
| 142 | + "J": 10, | |
| 143 | + "estimate_minutes_weighted": -0.0411522633744856, | |
| 144 | + "holm_p": NaN | |
| 145 | + }, | |
| 146 | + { | |
| 147 | + "contrast": "A3-A1", | |
| 148 | + "kind": "primary", | |
| 149 | + "outcome": "quality", | |
| 150 | + "estimate": 3.3472222222222205, | |
| 151 | + "ci_lo": -0.8055555555555592, | |
| 152 | + "ci_hi": 9.555555555555554, | |
| 153 | + "perm_p": 0.29985, | |
| 154 | + "J": 10, | |
| 155 | + "estimate_minutes_weighted": 4.571330589849107, | |
| 156 | + "holm_p": NaN | |
| 157 | + }, | |
| 158 | + { | |
| 159 | + "contrast": "A1-A0", | |
| 160 | + "kind": "secondary", | |
| 161 | + "outcome": "S", | |
| 162 | + "estimate": 0.8666666666666666, | |
| 163 | + "ci_lo": 0.6666666666666666, | |
| 164 | + "ci_hi": 1.0, | |
| 165 | + "perm_p": 0.0041, | |
| 166 | + "J": 10, | |
| 167 | + "estimate_minutes_weighted": 0.8847736625514403, | |
| 168 | + "holm_p": 0.012300000000000002 | |
| 169 | + }, | |
| 170 | + { | |
| 171 | + "contrast": "A1-A0", | |
| 172 | + "kind": "secondary", | |
| 173 | + "outcome": "D", | |
| 174 | + "estimate": 0.9666666666666666, | |
| 175 | + "ci_lo": 0.8999999999999998, | |
| 176 | + "ci_hi": 1.0, | |
| 177 | + "perm_p": 0.00225, | |
| 178 | + "J": 10, | |
| 179 | + "estimate_minutes_weighted": 0.9588477366255144, | |
| 180 | + "holm_p": NaN | |
| 181 | + }, | |
| 182 | + { | |
| 183 | + "contrast": "A1-A0", | |
| 184 | + "kind": "secondary", | |
| 185 | + "outcome": "E", | |
| 186 | + "estimate": -0.16666666666666669, | |
| 187 | + "ci_lo": -0.4666666666666667, | |
| 188 | + "ci_hi": 0.06666666666666667, | |
| 189 | + "perm_p": 0.49935, | |
| 190 | + "J": 10, | |
| 191 | + "estimate_minutes_weighted": -0.1440329218106996, | |
| 192 | + "holm_p": NaN | |
| 193 | + }, | |
| 194 | + { | |
| 195 | + "contrast": "A1-A0", | |
| 196 | + "kind": "secondary", | |
| 197 | + "outcome": "quality", | |
| 198 | + "estimate": 54.95833333333333, | |
| 199 | + "ci_lo": 40.44375, | |
| 200 | + "ci_hi": 68.23611111111111, | |
| 201 | + "perm_p": 0.0019, | |
| 202 | + "J": 10, | |
| 203 | + "estimate_minutes_weighted": 57.65946502057613, | |
| 204 | + "holm_p": NaN | |
| 205 | + }, | |
| 206 | + { | |
| 207 | + "contrast": "A4-A3", | |
| 208 | + "kind": "secondary", | |
| 209 | + "outcome": "S", | |
| 210 | + "estimate": -0.06666666666666668, | |
| 211 | + "ci_lo": -0.16666666666666669, | |
| 212 | + "ci_hi": 0.0, | |
| 213 | + "perm_p": 0.499, | |
| 214 | + "J": 10, | |
| 215 | + "estimate_minutes_weighted": -0.08230452674897121, | |
| 216 | + "holm_p": 0.998 | |
| 217 | + }, | |
| 218 | + { | |
| 219 | + "contrast": "A4-A3", | |
| 220 | + "kind": "secondary", | |
| 221 | + "outcome": "D", | |
| 222 | + "estimate": -0.03333333333333334, | |
| 223 | + "ci_lo": -0.1, | |
| 224 | + "ci_hi": 0.0, | |
| 225 | + "perm_p": 1.0, | |
| 226 | + "J": 10, | |
| 227 | + "estimate_minutes_weighted": -0.041152263374485604, | |
| 228 | + "holm_p": NaN | |
| 229 | + }, | |
| 230 | + { | |
| 231 | + "contrast": "A4-A3", | |
| 232 | + "kind": "secondary", | |
| 233 | + "outcome": "E", | |
| 234 | + "estimate": 0.0, | |
| 235 | + "ci_lo": 0.0, | |
| 236 | + "ci_hi": 0.0, | |
| 237 | + "perm_p": 1.0, | |
| 238 | + "J": 10, | |
| 239 | + "estimate_minutes_weighted": 0.0, | |
| 240 | + "holm_p": NaN | |
| 241 | + }, | |
| 242 | + { | |
| 243 | + "contrast": "A4-A3", | |
| 244 | + "kind": "secondary", | |
| 245 | + "outcome": "quality", | |
| 246 | + "estimate": -0.9027777777777757, | |
| 247 | + "ci_lo": -2.7086805555555578, | |
| 248 | + "ci_hi": 0.9725694444444483, | |
| 249 | + "perm_p": 0.39515, | |
| 250 | + "J": 10, | |
| 251 | + "estimate_minutes_weighted": -1.392318244170095, | |
| 252 | + "holm_p": NaN | |
| 253 | + }, | |
| 254 | + { | |
| 255 | + "contrast": "A2-A1", | |
| 256 | + "kind": "secondary", | |
| 257 | + "outcome": "S", | |
| 258 | + "estimate": 0.03333333333333334, | |
| 259 | + "ci_lo": -0.06666666666666668, | |
| 260 | + "ci_hi": 0.13333333333333336, | |
| 261 | + "perm_p": 1.0, | |
| 262 | + "J": 10, | |
| 263 | + "estimate_minutes_weighted": 0.024691358024691353, | |
| 264 | + "holm_p": 1.0 | |
| 265 | + }, | |
| 266 | + { | |
| 267 | + "contrast": "A2-A1", | |
| 268 | + "kind": "secondary", | |
| 269 | + "outcome": "D", | |
| 270 | + "estimate": 0.0, | |
| 271 | + "ci_lo": -0.1, | |
| 272 | + "ci_hi": 0.1, | |
| 273 | + "perm_p": 1.0, | |
| 274 | + "J": 10, | |
| 275 | + "estimate_minutes_weighted": 0.0, | |
| 276 | + "holm_p": NaN | |
| 277 | + }, | |
| 278 | + { | |
| 279 | + "contrast": "A2-A1", | |
| 280 | + "kind": "secondary", | |
| 281 | + "outcome": "E", | |
| 282 | + "estimate": -0.03333333333333333, | |
| 283 | + "ci_lo": -0.1, | |
| 284 | + "ci_hi": 0.0, | |
| 285 | + "perm_p": 1.0, | |
| 286 | + "J": 10, | |
| 287 | + "estimate_minutes_weighted": -0.0411522633744856, | |
| 288 | + "holm_p": NaN | |
| 289 | + }, | |
| 290 | + { | |
| 291 | + "contrast": "A2-A1", | |
| 292 | + "kind": "secondary", | |
| 293 | + "outcome": "quality", | |
| 294 | + "estimate": 2.625000000000001, | |
| 295 | + "ci_lo": -1.75, | |
| 296 | + "ci_hi": 8.792361111111102, | |
| 297 | + "perm_p": 0.5335, | |
| 298 | + "J": 10, | |
| 299 | + "estimate_minutes_weighted": 2.9646776406035653, | |
| 300 | + "holm_p": NaN | |
| 301 | + } | |
| 302 | + ], | |
| 303 | + "mixed_logit_S": { | |
| 304 | + "model": "cluster-robust logit (arm dummies, no intercept), clustered by scenario", | |
| 305 | + "params": { | |
| 306 | + "A0": { | |
| 307 | + "logit": -23.56606852345012, | |
| 308 | + "se_cluster": 0.33789975617634327, | |
| 309 | + "p": 5.82621463897232e-11 | |
| 310 | + }, | |
| 311 | + "A1": { | |
| 312 | + "logit": 1.8718021769015918, | |
| 313 | + "se_cluster": 0.8933372364778842, | |
| 314 | + "p": 0.8666666666666667 | |
| 315 | + }, | |
| 316 | + "A2": { | |
| 317 | + "logit": 2.1972245773362182, | |
| 318 | + "se_cluster": 0.8013385623960502, | |
| 319 | + "p": 0.8999999999999999 | |
| 320 | + }, | |
| 321 | + "A3": { | |
| 322 | + "logit": 2.1972245773362196, | |
| 323 | + "se_cluster": 1.1263325205221877, | |
| 324 | + "p": 0.8999999999999999 | |
| 325 | + }, | |
| 326 | + "A4": { | |
| 327 | + "logit": 1.609437912434101, | |
| 328 | + "se_cluster": 0.7476676377587332, | |
| 329 | + "p": 0.8333333333333334 | |
| 330 | + } | |
| 331 | + } | |
| 332 | + }, | |
| 333 | + "survival": { | |
| 334 | + "A0": { | |
| 335 | + "median_s": 153.3, | |
| 336 | + "n": 30, | |
| 337 | + "censored": 0, | |
| 338 | + "p_done_600": 1.0, | |
| 339 | + "p_done_1200": 1.0 | |
| 340 | + }, | |
| 341 | + "A1": { | |
| 342 | + "median_s": 216.8, | |
| 343 | + "n": 30, | |
| 344 | + "censored": 0, | |
| 345 | + "p_done_600": 1.0, | |
| 346 | + "p_done_1200": 1.0 | |
| 347 | + }, | |
| 348 | + "A2": { | |
| 349 | + "median_s": 213.8, | |
| 350 | + "n": 30, | |
| 351 | + "censored": 0, | |
| 352 | + "p_done_600": 1.0, | |
| 353 | + "p_done_1200": 1.0 | |
| 354 | + }, | |
| 355 | + "A3": { | |
| 356 | + "median_s": 217.8, | |
| 357 | + "n": 30, | |
| 358 | + "censored": 0, | |
| 359 | + "p_done_600": 1.0, | |
| 360 | + "p_done_1200": 1.0 | |
| 361 | + }, | |
| 362 | + "A4": { | |
| 363 | + "median_s": 228.8, | |
| 364 | + "n": 30, | |
| 365 | + "censored": 0, | |
| 366 | + "p_done_600": 1.0, | |
| 367 | + "p_done_1200": 1.0 | |
| 368 | + } | |
| 369 | + }, | |
| 370 | + "recovery": { | |
| 371 | + "n_A4": 30, | |
| 372 | + "n_failed_first_selfcheck": 0, | |
| 373 | + "n_recovery_attempted": 0, | |
| 374 | + "n_recovered": 0, | |
| 375 | + "ARI": null, | |
| 376 | + "acceptance_given_recovery": null, | |
| 377 | + "mean_recovery_cost": 0.0 | |
| 378 | + }, | |
| 379 | + "agreement": { | |
| 380 | + "krippendorff_alpha_interval": 0.9730624740153422, | |
| 381 | + "pairwise_pearson": { | |
| 382 | + "0": { | |
| 383 | + "0": 1.0, | |
| 384 | + "1": 0.964, | |
| 385 | + "2": 0.966 | |
| 386 | + }, | |
| 387 | + "1": { | |
| 388 | + "0": 0.964, | |
| 389 | + "1": 1.0, | |
| 390 | + "2": 0.99 | |
| 391 | + }, | |
| 392 | + "2": { | |
| 393 | + "0": 0.966, | |
| 394 | + "1": 0.99, | |
| 395 | + "2": 1.0 | |
| 396 | + } | |
| 397 | + }, | |
| 398 | + "n_runs": 150 | |
| 399 | + }, | |
| 400 | + "models": [ | |
| 401 | + "claude-opus-5-5" | |
| 402 | + ], | |
| 403 | + "claude_code_versions": [] | |
| 404 | +} | |
| \ No newline at end of file | ||
added
analysis/derived/recovery_experiment.json
+18 −0
@@ -0,0 +1,18 @@ | ||
| 1 | +{ | |
| 2 | + "ari": { | |
| 3 | + "n_A4_interrupted": 20, | |
| 4 | + "n_recovery_attempted": 20, | |
| 5 | + "n_selfcheck_pass_final": 20, | |
| 6 | + "ARI": 1.0, | |
| 7 | + "acceptance_after_recovery": 0.8421052631578947, | |
| 8 | + "recovery_cost_mean": 0.8632354999999998, | |
| 9 | + "recovery_wall_mean": 227.33 | |
| 10 | + }, | |
| 11 | + "contrast_A4int_minus_A3int_S": { | |
| 12 | + "estimate": 0.8, | |
| 13 | + "ci_lo": 0.55, | |
| 14 | + "ci_hi": 0.95, | |
| 15 | + "perm_p": 0.00415, | |
| 16 | + "J": 10 | |
| 17 | + } | |
| 18 | +} | |
| \ No newline at end of file | ||
modified
analysis/econometrics/estimate.py
+21 −1
@@ -12,12 +12,32 @@ CONTRASTS = [("A3", "A1", "primary"), ("A1", "A0", "secondary"), ("A4", "A3", "s | ||
| 12 | 12 | |
| 13 | 13 | def load(tag, q): |
| 14 | 14 | df = pd.read_parquet(os.path.join(REPO, "data", "validated", f"runs_{tag}.parquet")) |
| 15 | − df = df[df["status"] == "evaluated"].copy() | |
| 15 | + df = df[df["status"].isin(["collected", "evaluated"])].copy() # metrics are computed before the run is marked evaluated | |
| 16 | 16 | df["S"] = ((df["det_completion"].astype(bool)) & (~df["critical_error"].astype(bool)) & (df["quality"].astype(float) >= q)).astype(float) |
| 17 | 17 | df.loc[df["quality"].isna(), "S"] = np.nan |
| 18 | 18 | df["D"] = df["det_completion"].astype(float); df["E"] = df["critical_error"].astype(float) |
| 19 | 19 | df["C"] = df["total_cost_usd"].astype(float); df["T"] = df["wall_total_s"].astype(float) |
| 20 | 20 | df["censored"] = df["timed_out"].astype(bool) |
| 21 | + # Killed sessions emit no `result` event, so their marginal cost is unobserved (cost 0 recorded). Impute it from the session's | |
| 22 | + # token counts with per-token prices fitted by least squares on the completed runs of the same model (pricing is linear). | |
| 23 | + tok = df["tokens"].apply(lambda x: json.loads(x) if isinstance(x, str) else (x or {})) | |
| 24 | + X = pd.DataFrame({"input": tok.apply(lambda t: t.get("input", 0)), "output": tok.apply(lambda t: t.get("output", 0)), | |
| 25 | + "cache_read": tok.apply(lambda t: t.get("cache_read", 0)), "cache_create": tok.apply(lambda t: t.get("cache_create", 0))}, index=df.index).astype(float) | |
| 26 | + df["cost_observed"] = df["C"]; df["cost_imputed_flag"] = False | |
| 27 | + # prices are fitted on the completed pilot runs of the same model (the recovery stage has no completed first session) | |
| 28 | + try: | |
| 29 | + ref = pd.read_parquet(os.path.join(REPO, "data", "validated", "runs_pilot.parquet")); ref = ref[ref["cost_usd"].fillna(0) > 0] | |
| 30 | + except Exception: ref = df[df["cost_usd"].fillna(0) > 0] # noqa: BLE001 | |
| 31 | + rtok = ref["tokens"].apply(lambda x: json.loads(x) if isinstance(x, str) else (x or {})) | |
| 32 | + RX = pd.DataFrame({"input": rtok.apply(lambda t: t.get("input", 0)), "output": rtok.apply(lambda t: t.get("output", 0)), "cache_read": rtok.apply(lambda t: t.get("cache_read", 0)), "cache_create": rtok.apply(lambda t: t.get("cache_create", 0))}, index=ref.index).astype(float) | |
| 33 | + for mdl, g in df.groupby("model"): | |
| 34 | + fit = ref[ref["model"] == mdl] | |
| 35 | + miss = g[(g["cost_usd"].fillna(0) == 0) & g["censored"]] | |
| 36 | + if len(fit) >= 5 and len(miss): | |
| 37 | + beta, *_ = np.linalg.lstsq(RX.loc[fit.index].values, fit["cost_usd"].values, rcond=None) | |
| 38 | + imput = X.loc[miss.index].values @ beta | |
| 39 | + df.loc[miss.index, "C"] = np.maximum(0, imput) + df.loc[miss.index, "recovery_cost_usd"].fillna(0).astype(float) | |
| 40 | + df.loc[miss.index, "cost_imputed_flag"] = True | |
| 21 | 41 | return df |
| 22 | 42 | |
| 23 | 43 | def task_means(df, col, arm): |
added
analysis/figures/out/fig03_acceptance_by_arm_pilot.pdf
+0 −0
Binary file not shown.
added
analysis/figures/out/fig03_acceptance_by_arm_pilot.png
+0 −0
Binary file not shown.
added
analysis/figures/out/fig04_alf_curves_pilot.pdf
+0 −0
Binary file not shown.
added
analysis/figures/out/fig04_alf_curves_pilot.png
+0 −0
Binary file not shown.
added
analysis/figures/out/fig05_cost_quality_pilot.pdf
+0 −0
Binary file not shown.
added
analysis/figures/out/fig05_cost_quality_pilot.png
+0 −0
Binary file not shown.
added
analysis/figures/out/fig06_dependency_pilot.pdf
+0 −0
Binary file not shown.
added
analysis/figures/out/fig06_dependency_pilot.png
+0 −0
Binary file not shown.
added
analysis/figures/out/fig07_tool_effects_pilot.pdf
+0 −0
Binary file not shown.
added
analysis/figures/out/fig07_tool_effects_pilot.png
+0 −0
Binary file not shown.
added
analysis/figures/out/fig08_failure_severity_pilot.pdf
+0 −0
Binary file not shown.
added
analysis/figures/out/fig08_failure_severity_pilot.png
+0 −0
Binary file not shown.
added
analysis/figures/out/fig09_recovery_pilot.pdf
+0 −0
Binary file not shown.
added
analysis/figures/out/fig09_recovery_pilot.png
+0 −0
Binary file not shown.
added
analysis/figures/out/fig10_supervision_pilot.pdf
+0 −0
Binary file not shown.
added
analysis/figures/out/fig10_supervision_pilot.png
+0 −0
Binary file not shown.
added
analysis/figures/out/fig11_time_cdf_pilot.pdf
+0 −0
Binary file not shown.
added
analysis/figures/out/fig11_time_cdf_pilot.png
+0 −0
Binary file not shown.
added
analysis/figures/out/fig12_judge_agreement_pilot.pdf
+0 −0
Binary file not shown.
added
analysis/figures/out/fig12_judge_agreement_pilot.png
+0 −0
Binary file not shown.
added
analysis/figures/out/fig13_model_sensitivity.pdf
+0 −0
Binary file not shown.
added
analysis/figures/out/fig13_model_sensitivity.png
+0 −0
Binary file not shown.
added
analysis/figures/out/fig14_occupation_heterogeneity_pilot.pdf
+0 −0
Binary file not shown.
added
analysis/figures/out/fig14_occupation_heterogeneity_pilot.png
+0 −0
Binary file not shown.
added
analysis/figures/out/fig15_economic_sensitivity_pilot.pdf
+0 −0
Binary file not shown.
added
analysis/figures/out/fig15_economic_sensitivity_pilot.png
+0 −0
Binary file not shown.
added
analysis/robustness/audit_pilot.py
+62 −0
@@ -0,0 +1,62 @@ | ||
| 1 | +"""Phase 5 audit material (Agent I input): failure anatomy, critical violations, tool-policy compliance (denied calls), abstentions, | |
| 2 | +portal usage, A0 deliverable substitutions, judge-vs-deterministic disagreement. Writes analysis/tables/audit_*.csv and prints a summary. | |
| 3 | +Usage: .venv/bin/python analysis/robustness/audit_pilot.py --tag pilot""" | |
| 4 | +from __future__ import annotations | |
| 5 | +import argparse, json, os, sqlite3, collections | |
| 6 | +import pandas as pd | |
| 7 | +REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")); TAB = os.path.join(REPO, "analysis", "tables") | |
| 8 | +DB = os.path.join(REPO, "data", "raw", "wp42.sqlite") | |
| 9 | + | |
| 10 | +def main(): | |
| 11 | + ap = argparse.ArgumentParser(); ap.add_argument("--tag", default="pilot"); a = ap.parse_args() | |
| 12 | + con = sqlite3.connect(DB); con.row_factory = sqlite3.Row | |
| 13 | + runs = {r["run_id"]: dict(r) for r in con.execute("SELECT * FROM runs WHERE stage=?", (a.tag,))} | |
| 14 | + rec = {r["run_id"]: json.loads(r["receipt"]) for r in con.execute("SELECT c.run_id, receipt FROM receipts c JOIN runs r USING(run_id) WHERE r.stage=?", (a.tag,))} | |
| 15 | + met = {r["run_id"]: json.loads(r["metrics"]) for r in con.execute("SELECT m.run_id, metrics FROM metrics m JOIN runs r USING(run_id) WHERE r.stage=?", (a.tag,))} | |
| 16 | + # 1. failure anatomy: failed criteria and violations per run | |
| 17 | + rows = [] | |
| 18 | + for rid, rc in rec.items(): | |
| 19 | + failed = [c["id"] for c in rc.get("required_criteria", []) if not c["passed"]] | |
| 20 | + viol = [v["id"] for v in rc.get("critical_violations", [])] | |
| 21 | + m = met.get(rid, {}); q = m.get("quality") | |
| 22 | + rows.append({"run_id": rid, "task_id": rc["task_id"], "arm": runs[rid]["arm"], "rep": runs[rid]["rep"], "det": rc["det_completion"], "det_score": rc["det_score"], "failed_criteria": ";".join(failed), "violations": ";".join(viol), | |
| 23 | + "quality": q, "accepted": bool(rc["det_completion"] and not viol and q is not None and q >= 70), "completion_status": m.get("completion_status"), "n_questions": m.get("n_questions"), "abstention_justified": (rc.get("abstention") or {}).get("justified"), "result": runs[rid]["result_subtype"], "turns": runs[rid]["num_turns"], "cost": m.get("total_cost_usd")}) | |
| 24 | + df = pd.DataFrame(rows).sort_values(["task_id", "arm", "rep"]); df.to_csv(os.path.join(TAB, f"audit_failures_{a.tag}.csv"), index=False) | |
| 25 | + # 2. violations by type × arm | |
| 26 | + vt = collections.Counter() | |
| 27 | + for r in rows: | |
| 28 | + for v in filter(None, r["violations"].split(";")): vt[(r["arm"], v)] += 1 | |
| 29 | + pd.DataFrame([{"arm": k[0], "violation": k[1], "n": n} for k, n in sorted(vt.items())]).to_csv(os.path.join(TAB, f"audit_violations_{a.tag}.csv"), index=False) | |
| 30 | + # 3. denied tool calls (policy enforcement) and tool families per arm from events | |
| 31 | + ev = pd.DataFrame([dict(r) for r in con.execute("SELECT e.run_id, e.event_type, e.tool_name, e.summary, e.error_class FROM events e JOIN runs r USING(run_id) WHERE r.stage=?", (a.tag,))]) | |
| 32 | + ev["arm"] = ev["run_id"].map(lambda x: runs[x]["arm"]) | |
| 33 | + denied = ev[ev["summary"].str.contains("denied", na=False) & (ev["event_type"] == "user")] | |
| 34 | + den = denied.groupby("arm").size().rename("denied_calls").reset_index(); den.to_csv(os.path.join(TAB, f"audit_denied_{a.tag}.csv"), index=False) | |
| 35 | + denied_examples = denied.groupby("arm")["summary"].apply(lambda s: s.str[:160].head(3).tolist()) | |
| 36 | + fam = collections.Counter() | |
| 37 | + for rid, m in met.items(): | |
| 38 | + for f, n in (m.get("by_family") or {}).items(): fam[(runs[rid]["arm"], f)] += n | |
| 39 | + pd.DataFrame([{"arm": k[0], "family": k[1], "calls": n} for k, n in sorted(fam.items())]).to_csv(os.path.join(TAB, f"audit_families_{a.tag}.csv"), index=False) | |
| 40 | + # 4. portal usage (A3/A4): submissions present | |
| 41 | + portal = [] | |
| 42 | + for rid in runs: | |
| 43 | + p = os.path.join(REPO, "data", "raw", "runs", rid, "out", "portal_submissions.json") | |
| 44 | + portal.append({"run_id": rid, "arm": runs[rid]["arm"], "portal_submission": os.path.exists(p)}) | |
| 45 | + pdf = pd.DataFrame(portal); pu = pdf.groupby("arm")["portal_submission"].mean().reset_index(); pu.to_csv(os.path.join(TAB, f"audit_portal_{a.tag}.csv"), index=False) | |
| 46 | + # 5. A0 substitutions: which files did A0 produce | |
| 47 | + a0 = [] | |
| 48 | + for rid in [r for r in runs if runs[r]["arm"] == "A0"]: | |
| 49 | + out = os.path.join(REPO, "data", "raw", "runs", rid, "out") | |
| 50 | + a0.append({"run_id": rid, "files": ";".join(sorted(os.listdir(out))) if os.path.isdir(out) else ""}) | |
| 51 | + pd.DataFrame(a0).to_csv(os.path.join(TAB, f"audit_a0_outputs_{a.tag}.csv"), index=False) | |
| 52 | + # 6. det vs judges | |
| 53 | + jd = df.copy(); jd["judge_pass"] = jd["quality"] >= 70 | |
| 54 | + ct = pd.crosstab(jd["det"], jd["judge_pass"]); ct.to_csv(os.path.join(TAB, f"audit_det_vs_judge_{a.tag}.csv")) | |
| 55 | + print("runs", len(df), "| accepted by arm:", df.groupby("arm")["accepted"].mean().round(2).to_dict()) | |
| 56 | + print("det fail by arm:", df.groupby("arm")["det"].apply(lambda s: int((~s.astype(bool)).sum())).to_dict()) | |
| 57 | + print("violations:", dict(vt)); print("denied calls by arm:", den.to_dict("records")); print("denied examples:", denied_examples.to_dict()) | |
| 58 | + print("portal submissions by arm:", pu.to_dict("records")); print("det vs judge>=70:\n", ct) | |
| 59 | + print("failed runs:\n", df[~df["det"].astype(bool) | (df["violations"] != "")][["run_id", "det_score", "failed_criteria", "violations", "quality", "completion_status"]].to_string()) | |
| 60 | + | |
| 61 | +if __name__ == "__main__": | |
| 62 | + main() | |
added
analysis/robustness/economic_sensitivity.py
+57 −0
@@ -0,0 +1,57 @@ | ||
| 1 | +"""Economic sensitivity (mandate §12/§14 fig. 15) and supervision proxy (fig. 10). | |
| 2 | +- Cost per accepted deliverable by arm under API price multipliers (0.5x, 1x, 2x, 4x) and with a fully-loaded overhead (controller + judges + node time at an assumed node cost). | |
| 3 | +- Autonomy-Adjusted Productivity: accepted deliverables per 100 USD of (API cost + wage × human minutes), human minutes = 5 min per unanswered question (assumption) + interventions, wage ∈ {40, 80, 120} CAD/h. | |
| 4 | +- Human-cost benchmark (ASSUMPTION, not measured): the scenario authors' professional_minutes_estimate × wage, for comparison only. | |
| 5 | +Outputs table11_economic_sensitivity.csv, table12_supervision.csv, fig15, fig10.""" | |
| 6 | +import os, sys, json | |
| 7 | +import numpy as np, pandas as pd | |
| 8 | +import matplotlib; matplotlib.use("Agg"); import matplotlib.pyplot as plt | |
| 9 | +REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")); sys.path.insert(0, os.path.join(REPO, "analysis", "econometrics")); sys.path.insert(0, os.path.join(REPO, "analysis", "figures")) | |
| 10 | +from estimate import load | |
| 11 | +from make_figures import COL, LABEL, ARMS, save | |
| 12 | +plt.rcParams.update({"font.family": "serif", "font.size": 9, "axes.spines.top": False, "axes.spines.right": False, "axes.grid": True, "grid.color": "#e6e6e6", "grid.linewidth": 0.5, "legend.frameon": False}) | |
| 13 | + | |
| 14 | +def main(): | |
| 15 | + df = load("pilot", 70); jd = pd.read_parquet(os.path.join(REPO, "data", "validated", "judgments_pilot.parquet")) | |
| 16 | + judge_cost = jd.groupby("run_id")["cost_usd"].sum(); df["judge_cost"] = df["run_id"].map(judge_cost).fillna(0) | |
| 17 | + tasks = {t: json.load(open(os.path.join(REPO, "benchmark", "scenarios", t, "task.json"))) for t in df.task_id.unique()} | |
| 18 | + df["prof_min"] = df.task_id.map({t: v["professional_minutes_estimate"] for t, v in tasks.items()}) | |
| 19 | + df["questions"] = df["n_questions"].fillna(0).astype(float); df["H_min"] = df["human_intervention_min"].astype(float) + 5.0 * df["questions"] | |
| 20 | + rows = [] | |
| 21 | + for arm in ARMS: | |
| 22 | + d = df[df.arm == arm]; acc = d["S"].sum() | |
| 23 | + for mult in (0.5, 1.0, 2.0, 4.0): | |
| 24 | + api = d["C"].sum() * mult | |
| 25 | + rows.append({"arm": arm, "price_multiplier": mult, "cost_per_accepted_api": api / acc if acc else np.nan, | |
| 26 | + "cost_per_accepted_loaded": (api + d["judge_cost"].sum() + 0.05 * d["wall_total_s"].sum() / 3600 * 1.0) / acc if acc else np.nan}) # +1 USD/node-hour assumption | |
| 27 | + for wage in (40, 80, 120): | |
| 28 | + spend = d["C"].sum() + wage / 60 * d["H_min"].sum() | |
| 29 | + rows.append({"arm": arm, "wage_cad_h": wage, "accepted_per_100usd_aap": 100 * acc / spend if spend else np.nan, "mean_H_min": d["H_min"].mean()}) | |
| 30 | + t11 = pd.DataFrame(rows); t11.to_csv(os.path.join(REPO, "analysis", "tables", "table11_economic_sensitivity_pilot.csv"), index=False) | |
| 31 | + sup = df.groupby("arm").agg(n=("S", "size"), abstention_rate=("abstention_status", lambda s: s.isin(["needs_clarification", "abstained"]).mean()), mean_questions=("questions", "mean"), | |
| 32 | + justified_abstention_rate=("abstention_justified", lambda s: pd.Series(s).fillna(False).astype(bool).mean()), mean_H_min=("H_min", "mean")).reset_index() | |
| 33 | + sup.to_csv(os.path.join(REPO, "analysis", "tables", "table12_supervision_pilot.csv"), index=False) | |
| 34 | + # human-cost benchmark (assumption) | |
| 35 | + hb = pd.DataFrame([{"task_id": t, "prof_minutes": v["professional_minutes_estimate"], **{f"human_cost_{w}": v["professional_minutes_estimate"] / 60 * w for w in (40, 80, 120)}} for t, v in tasks.items()]) | |
| 36 | + hb.to_csv(os.path.join(REPO, "analysis", "tables", "table13_human_cost_assumption.csv"), index=False) | |
| 37 | + # fig15: cost per accepted deliverable vs price multiplier | |
| 38 | + fig, ax = plt.subplots(figsize=(5.0, 3.0)) | |
| 39 | + for arm in ARMS: | |
| 40 | + d = t11[(t11.arm == arm) & t11["price_multiplier"].notna()] | |
| 41 | + if d["cost_per_accepted_api"].notna().any(): ax.plot(d["price_multiplier"], d["cost_per_accepted_api"], marker="o", color=COL[arm], label=LABEL[arm], linewidth=1.6) | |
| 42 | + ax.set_xscale("log"); ax.set_xticks([0.5, 1, 2, 4]); ax.set_xticklabels(["0.5×", "1×", "2×", "4×"]); ax.set_xlabel("API price multiplier (relative to November 2026 list prices)"); ax.set_ylabel("marginal API cost per accepted deliverable (USD)") | |
| 43 | + ax.axhspan(hb["human_cost_40"].min(), hb["human_cost_120"].max(), color="#c3c2b7", alpha=0.25, lw=0); ax.text(0.52, hb["human_cost_40"].min() * 1.05, "authors' professional-time estimate × 40–120 CAD/h (assumption)", fontsize=6, color="#52514e") | |
| 44 | + ax.set_yscale("log"); ax.legend(fontsize=7); save(fig, "fig15_economic_sensitivity_pilot") | |
| 45 | + # fig10: supervision proxy | |
| 46 | + fig, ax = plt.subplots(figsize=(4.6, 2.8)); x = np.arange(len(sup)) | |
| 47 | + ax.bar(x - 0.2, sup["abstention_rate"], width=0.4, color=[COL[a] for a in sup.arm], label="runs flagging questions / abstaining") | |
| 48 | + ax.bar(x + 0.2, sup["justified_abstention_rate"], width=0.4, facecolor="none", edgecolor="#444", linewidth=0.9, label="of which justified by the scenario design") | |
| 49 | + ax.set_xticks(x); ax.set_xticklabels(sup.arm); ax.set_ylabel("share of runs"); ax.set_ylim(0, 1.05); ax.legend(fontsize=7, loc="upper right"); save(fig, "fig10_supervision_pilot") | |
| 50 | + # fig13 (replaces the perturbation figure, not run): model sensitivity on A3 | |
| 51 | + t9 = pd.read_csv(os.path.join(REPO, "analysis", "tables", "table9_model_sensitivity.csv")) | |
| 52 | + fig, ax = plt.subplots(figsize=(4.6, 2.8)); cols = {"sonnet": "#2a78d6", "opus": "#eb6834", "fable": "#1baf7a"} | |
| 53 | + for _, r in t9.iterrows(): | |
| 54 | + ax.scatter(r["cost_mean"], r["acceptance_taskw"], s=90, color=cols.get(r["model"], "#444"), edgecolor="#222"); ax.annotate(f"{r['model']}\nQ={r['quality_mean']:.0f}", (r["cost_mean"], r["acceptance_taskw"]), textcoords="offset points", xytext=(6, -14), fontsize=7) | |
| 55 | + ax.set_xscale("log"); ax.set_xlabel("mean marginal API cost per run (USD, log)"); ax.set_ylabel("acceptance rate (arm A3)"); ax.set_ylim(0.5, 1.0); save(fig, "fig13_model_sensitivity") | |
| 56 | + print(t11.round(3).to_string()); print(sup.round(3).to_string()) | |
| 57 | +if __name__ == "__main__": main() | |
added
analysis/robustness/recovery_experiment.py
+29 −0
@@ -0,0 +1,29 @@ | ||
| 1 | +"""Exploratory recovery experiment (A-011): A3 vs A4 with the first session interrupted at 120 s. | |
| 2 | +ARI = P(final self-check pass | interrupted, A4); acceptance after recovery vs the interrupted A3 control and vs uninterrupted A3/A4 (pilot).""" | |
| 3 | +import os, sys, json | |
| 4 | +import numpy as np, pandas as pd | |
| 5 | +REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")); sys.path.insert(0, os.path.join(REPO, "analysis", "econometrics")) | |
| 6 | +from estimate import load, cluster_bootstrap, permutation_p | |
| 7 | +def main(): | |
| 8 | + rec = load("recovery", 70); pilot = load("pilot", 70) | |
| 9 | + rows = [] | |
| 10 | + for arm, g in rec.groupby("arm"): | |
| 11 | + acc = g["S"].sum() | |
| 12 | + rows.append({"condition": f"{arm} interrupted@120s", "n": len(g), "acceptance": g.groupby("task_id")["S"].mean().mean(), "det": g["D"].mean(), "quality": g["quality"].mean(), | |
| 13 | + "critical": g["E"].mean(), "cost_mean": g["C"].mean(), "cost_per_accepted": g["C"].sum() / acc if acc else np.nan, "wall_total_median": g["wall_total_s"].median(), | |
| 14 | + "recovery_attempted": g["recovery_attempted"].astype(bool).mean(), "selfcheck_first": g["selfcheck_pass_first"].mean(), "selfcheck_final": g["selfcheck_pass_final"].mean()}) | |
| 15 | + for arm in ("A3", "A4"): | |
| 16 | + g = pilot[pilot.arm == arm]; acc = g["S"].sum() | |
| 17 | + rows.append({"condition": f"{arm} uninterrupted (pilot)", "n": len(g), "acceptance": g.groupby("task_id")["S"].mean().mean(), "det": g["D"].mean(), "quality": g["quality"].mean(), "critical": g["E"].mean(), | |
| 18 | + "cost_mean": g["C"].mean(), "cost_per_accepted": g["C"].sum() / acc if acc else np.nan, "wall_total_median": g["wall_total_s"].median(), "recovery_attempted": g["recovery_attempted"].astype(bool).mean(), | |
| 19 | + "selfcheck_first": g["selfcheck_pass_first"].mean(), "selfcheck_final": g["selfcheck_pass_final"].mean()}) | |
| 20 | + t = pd.DataFrame(rows); t.to_csv(os.path.join(REPO, "analysis", "tables", "table10_recovery.csv"), index=False) | |
| 21 | + a4 = rec[rec.arm == "A4"]; interrupted = a4[a4["recovery_attempted"].astype(bool)] | |
| 22 | + ari = {"n_A4_interrupted": int(len(a4)), "n_recovery_attempted": int(len(interrupted)), "n_selfcheck_pass_final": int((interrupted["selfcheck_pass_final"] == 1).sum()), | |
| 23 | + "ARI": float((interrupted["selfcheck_pass_final"] == 1).mean()) if len(interrupted) else None, "acceptance_after_recovery": float(interrupted["S"].mean()) if len(interrupted) else None, | |
| 24 | + "recovery_cost_mean": float(a4["recovery_cost_usd"].fillna(0).mean()), "recovery_wall_mean": float(a4.get("wall_total_s", pd.Series()).mean() if len(a4) else 0)} | |
| 25 | + rng = np.random.default_rng(7); est, lo, hi, J = cluster_bootstrap(rec, "S", "A4", "A3", 5000, rng); p = permutation_p(rec, "S", "A4", "A3", 20000, rng) | |
| 26 | + out = {"ari": ari, "contrast_A4int_minus_A3int_S": {"estimate": est, "ci_lo": lo, "ci_hi": hi, "perm_p": p, "J": J}} | |
| 27 | + json.dump(out, open(os.path.join(REPO, "analysis", "derived", "recovery_experiment.json"), "w"), indent=1) | |
| 28 | + print(t.round(3).to_string()); print(json.dumps(out, indent=1)) | |
| 29 | +if __name__ == "__main__": main() | |
added
analysis/robustness/sensitivity_models.py
+28 −0
@@ -0,0 +1,28 @@ | ||
| 1 | +"""Model sensitivity: arm A3 under claude-opus-5-5 (pilot, R=3) vs claude-sonnet-5-5 and claude-fable-5-1 (stage sensitivity, R=2). | |
| 2 | +Outputs analysis/tables/table9_model_sensitivity.csv and per-scenario table.""" | |
| 3 | +import os, sys, json | |
| 4 | +import numpy as np, pandas as pd | |
| 5 | +REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")); sys.path.insert(0, os.path.join(REPO, "analysis", "econometrics")) | |
| 6 | +from estimate import load, cluster_bootstrap, permutation_p | |
| 7 | +def main(): | |
| 8 | + pilot = load("pilot", 70); sens = load("sensitivity", 70) | |
| 9 | + a3 = pilot[pilot.arm == "A3"].copy(); a3["model_tag"] = "opus" | |
| 10 | + sens["model_tag"] = sens["model"].map({"claude-sonnet-5-5": "sonnet", "claude-fable-5-1": "fable"}) | |
| 11 | + df = pd.concat([a3, sens]); rows = [] | |
| 12 | + for m, g in df.groupby("model_tag"): | |
| 13 | + acc = g["S"].sum() | |
| 14 | + rows.append({"model": m, "n_runs": len(g), "acceptance_taskw": g.groupby("task_id")["S"].mean().mean(), "det_completion": g["D"].mean(), "critical_error_rate": g["E"].mean(), | |
| 15 | + "quality_mean": g["quality"].mean(), "cost_mean": g["C"].mean(), "cost_per_accepted": g["C"].sum() / acc if acc else np.nan, "wall_median_s": g["T"].median(), | |
| 16 | + "tool_calls_mean": g["tool_calls"].mean(), "timeout_rate": g["censored"].mean()}) | |
| 17 | + t = pd.DataFrame(rows); t.to_csv(os.path.join(REPO, "analysis", "tables", "table9_model_sensitivity.csv"), index=False) | |
| 18 | + # contrasts vs opus (scenario-cluster bootstrap on S and quality); reuse helpers by relabelling arm := model_tag | |
| 19 | + d2 = df.copy(); d2["arm"] = d2["model_tag"]; rng = np.random.default_rng(42); crows = [] | |
| 20 | + for m in ("sonnet", "fable"): | |
| 21 | + for col in ("S", "D", "quality", "C"): | |
| 22 | + est, lo, hi, J = cluster_bootstrap(d2, col, m, "opus", 5000, rng); p = permutation_p(d2, col, m, "opus", 20000, rng) | |
| 23 | + crows.append({"contrast": f"{m}-opus", "outcome": col, "estimate": est, "ci_lo": lo, "ci_hi": hi, "perm_p": p, "J": J}) | |
| 24 | + c = pd.DataFrame(crows); c.to_csv(os.path.join(REPO, "analysis", "tables", "table9b_model_contrasts.csv"), index=False) | |
| 25 | + per = df.groupby(["task_id", "model_tag"]).agg(acceptance=("S", "mean"), quality=("quality", "mean"), cost=("C", "mean")).reset_index() | |
| 26 | + per.to_csv(os.path.join(REPO, "analysis", "tables", "table9c_model_by_scenario.csv"), index=False) | |
| 27 | + print(t.round(3).to_string()); print(c.round(3).to_string()) | |
| 28 | +if __name__ == "__main__": main() | |
added
analysis/tables/audit_a0_outputs_pilot.csv
+31 −0
@@ -0,0 +1,31 @@ | ||
| 1 | +run_id,files | |
| 2 | +PROC-01_A0_r1,COMPLETION.json;bid_evaluation.md;bid_evaluation.xml;recommendation_memo.md | |
| 3 | +RES-01_A0_r1,COMPLETION.json;code;reference_check.csv;report.md | |
| 4 | +ACC-01_A0_r2,COMPLETION.json;management_report.md;reconciliation_BankDetail.csv;reconciliation_Discrepancies.csv;reconciliation_Monthly.csv | |
| 5 | +ADM-01_A0_r2,COMPLETION.json;action_plan.md;action_plan_Calendar.csv;action_plan_Tasks.csv;briefing_note.md;filing;replies | |
| 6 | +HR-01_A0_r2,COMPLETION.json;hr_summary.md;letters;payroll_adjustments.csv;payroll_adjustments_weekly_hours.csv;test_probe.txt | |
| 7 | +RE-01_A0_r2,COMPLETION.json;adjustment_grid.csv;appraisal_report.md | |
| 8 | +PROC-01_A0_r2,COMPLETION.json;bid_evaluation.xml;recommendation_memo.md;test.txt | |
| 9 | +RE-01_A0_r1,COMPLETION.json;comparable_screening.md | |
| 10 | +BI-01_A0_r2,COMPLETION.json;code | |
| 11 | +LEGAL-01_A0_r2,COMPLETION.json;issues_register.csv;review_memo.md | |
| 12 | +ADM-01_A0_r1,COMPLETION.json;action_plan_Calendar.csv;action_plan_Tasks.csv;briefing_note.md;filing;replies | |
| 13 | +HR-01_A0_r1,COMPLETION.json;hr_summary.md;letters;payroll_adjustments.csv;payroll_adjustments_weekly_hours.csv;test_probe.md | |
| 14 | +FIN-01_A0_r1,COMPLETION.json;dcf_model.md;investment_memo.md | |
| 15 | +ACC-01_A0_r1,COMPLETION.json;management_report.md;reconciliation.csv;reconciliation_Discrepancies.csv | |
| 16 | +BI-01_A0_r1,COMPLETION.json;code | |
| 17 | +BANK-01_A0_r2,COMPLETION.json;credit_memo.md;ratio_analysis.csv | |
| 18 | +RES-01_A0_r2,COMPLETION.json;code;report.md;tables | |
| 19 | +BANK-01_A0_r1,COMPLETION.json;credit_memo.md;ratio_analysis.csv | |
| 20 | +LEGAL-01_A0_r1,COMPLETION.json;issues_register.csv;review_memo.md | |
| 21 | +FIN-01_A0_r2,COMPLETION.json;dcf_model.md;investment_memo.md | |
| 22 | +LEGAL-01_A0_r3,COMPLETION.json;issues_register.csv;playbook_checklist.csv;review_memo.md | |
| 23 | +FIN-01_A0_r3,COMPLETION.json;investment_memo.md | |
| 24 | +RE-01_A0_r3,COMPLETION.json;adjustment_grid.csv;appraisal_report.md | |
| 25 | +PROC-01_A0_r3,COMPLETION.json;bid_evaluation.csv;recommendation_memo.md | |
| 26 | +ADM-01_A0_r3,COMPLETION.json;action_plan.md;briefing_note.md;filing;replies | |
| 27 | +BANK-01_A0_r3,COMPLETION.json;credit_memo.md;ratio_analysis.csv | |
| 28 | +RES-01_A0_r3,COMPLETION.json;code;report.md | |
| 29 | +BI-01_A0_r3,COMPLETION.json;code | |
| 30 | +HR-01_A0_r3,COMPLETION.json;hr_summary.md;letters;payroll_adjustments.csv;payroll_adjustments_weekly_hours.csv;test_probe.md | |
| 31 | +ACC-01_A0_r3,COMPLETION.json;management_report.md;reconciliation_BankItems.csv;reconciliation_Discrepancies.csv;reconciliation_Monthly.csv | |
added
analysis/tables/audit_denied_pilot.csv
+4 −0
@@ -0,0 +1,4 @@ | ||
| 1 | +arm,denied_calls | |
| 2 | +A1,1 | |
| 3 | +A2,1 | |
| 4 | +A4,1 | |
added
analysis/tables/audit_det_vs_judge_pilot.csv
+3 −0
@@ -0,0 +1,3 @@ | ||
| 1 | +det,False,True | |
| 2 | +False,31,2 | |
| 3 | +True,12,105 | |
added
analysis/tables/audit_failures_pilot.csv
+151 −0
@@ -0,0 +1,151 @@ | ||
| 1 | +run_id,task_id,arm,rep,det,det_score,failed_criteria,violations,quality,accepted,completion_status,n_questions,abstention_justified,result,turns,cost | |
| 2 | +ACC-01_A0_r1,ACC-01,A0,1,False,0.1667,xlsx_structure;xlsx_formulas;docx_report;discrepancies_coverage;reconciled_balance_dec;total_adjustment;cash_position_three_figures,,29.583333333333332,False,needs_clarification,3,False,success,32,0.6473212 | |
| 3 | +ACC-01_A0_r2,ACC-01,A0,2,False,0.1667,xlsx_structure;xlsx_formulas;docx_report;discrepancies_coverage;reconciled_balance_dec;total_adjustment;cash_position_three_figures,,25.416666666666668,False,needs_clarification,4,False,success,35,0.7746372 | |
| 4 | +ACC-01_A0_r3,ACC-01,A0,3,False,0.1667,xlsx_structure;xlsx_formulas;docx_report;discrepancies_coverage;reconciled_balance_dec;total_adjustment;cash_position_three_figures,,26.25,False,needs_clarification,4,False,success,39,0.8091436000000001 | |
| 5 | +ACC-01_A1_r1,ACC-01,A1,1,True,1.0,,,95.41666666666667,True,completed,3,,success,28,1.3541987999999998 | |
| 6 | +ACC-01_A1_r2,ACC-01,A1,2,True,1.0,,,97.5,True,completed,1,,success,19,1.0732232000000002 | |
| 7 | +ACC-01_A1_r3,ACC-01,A1,3,True,1.0,,,92.91666666666667,True,completed,2,,success,21,0.9608876000000001 | |
| 8 | +ACC-01_A2_r1,ACC-01,A2,1,True,1.0,,,98.75,True,completed,2,,success,21,1.2220724 | |
| 9 | +ACC-01_A2_r2,ACC-01,A2,2,True,1.0,,,93.75,True,completed,2,,success,19,0.9797699999999999 | |
| 10 | +ACC-01_A2_r3,ACC-01,A2,3,True,1.0,,,75.41666666666667,True,completed,0,,success,21,0.9150716 | |
| 11 | +ACC-01_A3_r1,ACC-01,A3,1,True,1.0,,,100.0,True,completed,2,,success,22,1.2184656000000003 | |
| 12 | +ACC-01_A3_r2,ACC-01,A3,2,True,1.0,,,96.66666666666667,True,completed,2,,success,25,1.1660313999999998 | |
| 13 | +ACC-01_A3_r3,ACC-01,A3,3,True,1.0,,,100.0,True,completed,3,,success,29,1.1707763999999998 | |
| 14 | +ACC-01_A4_r1,ACC-01,A4,1,True,1.0,,,96.66666666666667,True,completed,2,,success,25,1.1326603999999998 | |
| 15 | +ACC-01_A4_r2,ACC-01,A4,2,True,1.0,,,90.41666666666667,True,completed,2,,success,29,1.1537353999999995 | |
| 16 | +ACC-01_A4_r3,ACC-01,A4,3,True,1.0,,,94.16666666666667,True,completed,3,,success,28,1.3211015999999998 | |
| 17 | +ADM-01_A0_r1,ADM-01,A0,1,False,0.4118,R1_deliverables;R3_conflicts;R4_top5;R7_briefing;R8_timezone;R9_travel,,47.083333333333336,False,completed,8,,success,81,1.5545744 | |
| 18 | +ADM-01_A0_r2,ADM-01,A0,2,False,0.4118,R1_deliverables;R3_conflicts;R4_top5;R7_briefing;R8_timezone;R9_travel,,51.25,False,completed,11,,success,80,1.6231584 | |
| 19 | +ADM-01_A0_r3,ADM-01,A0,3,False,0.4118,R1_deliverables;R3_conflicts;R4_top5;R7_briefing;R8_timezone;R9_travel,,55.833333333333336,False,needs_clarification,9,True,success,76,1.5290284 | |
| 20 | +ADM-01_A1_r1,ADM-01,A1,1,True,1.0,,,64.16666666666667,False,completed,10,,success,16,1.2258108 | |
| 21 | +ADM-01_A1_r2,ADM-01,A1,2,True,1.0,,,61.25,False,completed,12,,success,19,1.2803403999999998 | |
| 22 | +ADM-01_A1_r3,ADM-01,A1,3,True,1.0,,,62.916666666666664,False,completed,7,,success,12,1.1743472 | |
| 23 | +ADM-01_A2_r1,ADM-01,A2,1,True,1.0,,,62.083333333333336,False,completed,9,,success,15,1.0714164 | |
| 24 | +ADM-01_A2_r2,ADM-01,A2,2,True,1.0,,,70.41666666666667,True,completed,7,,success,10,1.0318544 | |
| 25 | +ADM-01_A2_r3,ADM-01,A2,3,True,1.0,,,67.5,False,completed,9,,success,16,1.1457846 | |
| 26 | +ADM-01_A3_r1,ADM-01,A3,1,True,1.0,,,59.583333333333336,False,completed,7,,success,19,1.4193666 | |
| 27 | +ADM-01_A3_r2,ADM-01,A3,2,True,1.0,,,65.83333333333333,False,completed,9,,success,20,1.4769808000000002 | |
| 28 | +ADM-01_A3_r3,ADM-01,A3,3,True,1.0,,,61.25000000000001,False,completed,8,,success,17,1.2574988 | |
| 29 | +ADM-01_A4_r1,ADM-01,A4,1,True,1.0,,,61.25,False,completed,8,,success,19,1.2477093999999997 | |
| 30 | +ADM-01_A4_r2,ADM-01,A4,2,True,1.0,,,65.83333333333333,False,completed,8,,success,20,1.2482252000000003 | |
| 31 | +ADM-01_A4_r3,ADM-01,A4,3,True,1.0,,,67.5,False,completed,7,,success,19,1.3489496 | |
| 32 | +BANK-01_A0_r1,BANK-01,A0,1,False,0.069,deliverables_present;xlsx_formulas;dscr_proforma;leverage_proforma_with_lease;risk_rating;recommendation_not_unconditional;revenue_discrepancy_flagged;lease_capitalisation;ltv;current_ratio_2025;memo_structure,,16.25,False,needs_clarification,3,False,success,26,0.47591620000000007 | |
| 33 | +BANK-01_A0_r2,BANK-01,A0,2,False,0.069,deliverables_present;xlsx_formulas;dscr_proforma;leverage_proforma_with_lease;risk_rating;recommendation_not_unconditional;revenue_discrepancy_flagged;lease_capitalisation;ltv;current_ratio_2025;memo_structure,,16.25,False,needs_clarification,3,False,success,21,0.5257702 | |
| 34 | +BANK-01_A0_r3,BANK-01,A0,3,False,0.069,deliverables_present;xlsx_formulas;dscr_proforma;leverage_proforma_with_lease;risk_rating;recommendation_not_unconditional;revenue_discrepancy_flagged;lease_capitalisation;ltv;current_ratio_2025;memo_structure,,17.5,False,needs_clarification,4,False,success,16,0.41576719999999995 | |
| 35 | +BANK-01_A1_r1,BANK-01,A1,1,True,1.0,,,95.0,True,completed,4,,success,10,0.8782425999999999 | |
| 36 | +BANK-01_A1_r2,BANK-01,A1,2,True,1.0,,,97.5,True,completed,3,,success,9,0.9325792000000002 | |
| 37 | +BANK-01_A1_r3,BANK-01,A1,3,True,1.0,,,97.5,True,completed,5,,success,12,1.032834 | |
| 38 | +BANK-01_A2_r1,BANK-01,A2,1,True,1.0,,,92.91666666666667,True,completed,4,,success,9,0.7360516 | |
| 39 | +BANK-01_A2_r2,BANK-01,A2,2,True,1.0,,,92.5,True,completed,4,,success,7,0.8021278000000001 | |
| 40 | +BANK-01_A2_r3,BANK-01,A2,3,True,1.0,,,97.5,True,completed,3,,success,9,0.8617402000000001 | |
| 41 | +BANK-01_A3_r1,BANK-01,A3,1,True,1.0,,,92.91666666666667,True,completed,4,,success,14,0.81718 | |
| 42 | +BANK-01_A3_r2,BANK-01,A3,2,True,1.0,,,92.5,True,completed,3,,success,12,0.8122035999999998 | |
| 43 | +BANK-01_A3_r3,BANK-01,A3,3,True,1.0,,,88.33333333333333,True,completed,3,,success,13,0.8980578000000001 | |
| 44 | +BANK-01_A4_r1,BANK-01,A4,1,True,1.0,,,89.58333333333333,True,completed,2,,success,15,0.8588318 | |
| 45 | +BANK-01_A4_r2,BANK-01,A4,2,True,1.0,,,93.33333333333333,True,completed,2,,success,16,0.9456506 | |
| 46 | +BANK-01_A4_r3,BANK-01,A4,3,True,1.0,,,91.66666666666667,True,completed,2,,success,18,0.8340269999999999 | |
| 47 | +BI-01_A0_r1,BI-01,A0,1,False,0.037,C01_deliverables;C02_structure_formulas;C03_chart;C04_net_revenue;C05_yoy;C06_gross_margin;C07_top10;C08_monthly;C09_regions;C10_data_quality;C11_commentary;C12_secondary_kpis;C14_on_time,,5.416666666666667,False,needs_clarification,2,False,success,14,0.9253386 | |
| 48 | +BI-01_A0_r2,BI-01,A0,2,False,0.037,C01_deliverables;C02_structure_formulas;C03_chart;C04_net_revenue;C05_yoy;C06_gross_margin;C07_top10;C08_monthly;C09_regions;C10_data_quality;C11_commentary;C12_secondary_kpis;C14_on_time,,4.583333333333333,False,needs_clarification,3,False,success,14,0.76611 | |
| 49 | +BI-01_A0_r3,BI-01,A0,3,False,0.037,C01_deliverables;C02_structure_formulas;C03_chart;C04_net_revenue;C05_yoy;C06_gross_margin;C07_top10;C08_monthly;C09_regions;C10_data_quality;C11_commentary;C12_secondary_kpis;C14_on_time,,6.25,False,needs_clarification,2,False,success,15,0.9048236 | |
| 50 | +BI-01_A1_r1,BI-01,A1,1,False,0.037,C01_deliverables;C02_structure_formulas;C03_chart;C04_net_revenue;C05_yoy;C06_gross_margin;C07_top10;C08_monthly;C09_regions;C10_data_quality;C11_commentary;C12_secondary_kpis;C14_on_time,inputs_modified_harness,3.75,False,needs_clarification,2,False,success,13,0.6842858000000002 | |
| 51 | +BI-01_A1_r2,BI-01,A1,2,True,1.0,,,73.33333333333333,True,completed,2,,success,18,0.9662652 | |
| 52 | +BI-01_A1_r3,BI-01,A1,3,True,1.0,,,72.91666666666667,True,completed,2,,success,14,0.9836679999999999 | |
| 53 | +BI-01_A2_r1,BI-01,A2,1,True,1.0,,,75.0,True,completed,2,,success,13,0.8454745999999999 | |
| 54 | +BI-01_A2_r2,BI-01,A2,2,True,1.0,,,80.41666666666667,True,completed,1,,success,19,1.0569582 | |
| 55 | +BI-01_A2_r3,BI-01,A2,3,True,1.0,,,75.41666666666667,True,completed,1,,success,17,0.9047606000000002 | |
| 56 | +BI-01_A3_r1,BI-01,A3,1,True,1.0,,,87.08333333333333,True,completed,2,,success,23,1.3251399999999998 | |
| 57 | +BI-01_A3_r2,BI-01,A3,2,True,1.0,,,74.16666666666667,True,completed,0,,success,22,0.9944262 | |
| 58 | +BI-01_A3_r3,BI-01,A3,3,True,1.0,,,72.91666666666667,True,completed,2,,success,26,1.3171925999999998 | |
| 59 | +BI-01_A4_r1,BI-01,A4,1,True,1.0,,,67.91666666666667,False,completed,1,,success,28,1.2259292 | |
| 60 | +BI-01_A4_r2,BI-01,A4,2,True,1.0,,,75.0,True,completed,2,,success,25,1.1139329999999996 | |
| 61 | +BI-01_A4_r3,BI-01,A4,3,True,1.0,,,74.16666666666667,True,completed,0,,success,22,1.1441234 | |
| 62 | +FIN-01_A0_r1,FIN-01,A0,1,False,0.0,deliverables_present;model_structure;wacc_value;enterprise_value;value_per_share;sensitivity_table;memo_content;outlier_beta_excluded;completion_valid;recommendation,,17.5,False,needs_clarification,4,False,success,15,0.348323 | |
| 63 | +FIN-01_A0_r2,FIN-01,A0,2,False,0.0,deliverables_present;model_structure;wacc_value;enterprise_value;value_per_share;sensitivity_table;memo_content;outlier_beta_excluded;completion_valid;recommendation,,20.0,False,needs_clarification,3,False,success,15,0.297777 | |
| 64 | +FIN-01_A0_r3,FIN-01,A0,3,False,0.0,deliverables_present;model_structure;wacc_value;enterprise_value;value_per_share;sensitivity_table;memo_content;outlier_beta_excluded;completion_valid;recommendation,,20.833333333333332,False,needs_clarification,4,False,success,18,0.2819248 | |
| 65 | +FIN-01_A1_r1,FIN-01,A1,1,True,1.0,,,83.75,True,completed,2,,success,15,0.8704069999999999 | |
| 66 | +FIN-01_A1_r2,FIN-01,A1,2,True,1.0,,,90.0,True,completed,1,,success,14,0.7455324 | |
| 67 | +FIN-01_A1_r3,FIN-01,A1,3,True,1.0,,,82.91666666666667,True,completed,2,,success,11,0.7689566 | |
| 68 | +FIN-01_A2_r1,FIN-01,A2,1,True,1.0,,,77.91666666666667,True,completed,1,,success,12,0.6988323999999999 | |
| 69 | +FIN-01_A2_r2,FIN-01,A2,2,True,1.0,,,82.5,True,completed,1,,success,12,0.6927183999999998 | |
| 70 | +FIN-01_A2_r3,FIN-01,A2,3,True,1.0,,,87.08333333333333,True,completed,2,,success,10,0.7991102 | |
| 71 | +FIN-01_A3_r1,FIN-01,A3,1,True,1.0,,,90.83333333333333,True,completed,1,,success,12,0.7088782 | |
| 72 | +FIN-01_A3_r2,FIN-01,A3,2,True,1.0,,,90.41666666666667,True,completed,1,,success,13,0.6977076000000001 | |
| 73 | +FIN-01_A3_r3,FIN-01,A3,3,True,1.0,,,82.08333333333333,True,completed,0,,success,15,0.6731418 | |
| 74 | +FIN-01_A4_r1,FIN-01,A4,1,True,1.0,,,83.75,True,completed,0,,success,15,0.6193339999999999 | |
| 75 | +FIN-01_A4_r2,FIN-01,A4,2,True,1.0,,,92.5,True,completed,0,,success,16,0.87707 | |
| 76 | +FIN-01_A4_r3,FIN-01,A4,3,True,1.0,,,82.5,True,completed,0,,success,15,0.6825034000000001 | |
| 77 | +HR-01_A0_r1,HR-01,A0,1,False,0.1765,deliverables_present;formulas_min;table_structure;overtime_hours;weekly_overtime_rule;holiday_eligibility;vacation_accrual;resignation_payout;missing_hire_date_question;letters_content,,37.083333333333336,False,needs_clarification,7,True,success,37,0.8788545999999999 | |
| 78 | +HR-01_A0_r2,HR-01,A0,2,False,0.1765,deliverables_present;formulas_min;table_structure;overtime_hours;weekly_overtime_rule;holiday_eligibility;vacation_accrual;resignation_payout;missing_hire_date_question;letters_content,,33.333333333333336,False,needs_clarification,7,True,success,58,0.8021521999999999 | |
| 79 | +HR-01_A0_r3,HR-01,A0,3,False,0.1765,deliverables_present;formulas_min;table_structure;overtime_hours;weekly_overtime_rule;holiday_eligibility;vacation_accrual;resignation_payout;missing_hire_date_question;letters_content,,35.833333333333336,False,needs_clarification,7,True,success,34,0.8275657999999999 | |
| 80 | +HR-01_A1_r1,HR-01,A1,1,True,1.0,,,72.91666666666667,True,needs_clarification,6,True,success,11,0.7251406 | |
| 81 | +HR-01_A1_r2,HR-01,A1,2,True,1.0,,,72.08333333333333,True,needs_clarification,5,True,success,9,0.7273792 | |
| 82 | +HR-01_A1_r3,HR-01,A1,3,True,1.0,,,75.83333333333333,True,needs_clarification,7,True,success,13,0.79254 | |
| 83 | +HR-01_A2_r1,HR-01,A2,1,True,1.0,,,76.66666666666667,True,needs_clarification,5,True,success,11,0.7625718 | |
| 84 | +HR-01_A2_r2,HR-01,A2,2,True,1.0,,,76.66666666666667,True,needs_clarification,6,True,success,13,0.8454119999999998 | |
| 85 | +HR-01_A2_r3,HR-01,A2,3,True,1.0,,,75.0,True,completed,5,,success,10,0.7553578 | |
| 86 | +HR-01_A3_r1,HR-01,A3,1,True,1.0,,,73.33333333333333,True,needs_clarification,4,True,success,14,0.7962018 | |
| 87 | +HR-01_A3_r2,HR-01,A3,2,True,1.0,,,75.83333333333333,True,needs_clarification,3,True,success,17,0.9144788000000001 | |
| 88 | +HR-01_A3_r3,HR-01,A3,3,True,1.0,,,79.58333333333333,True,needs_clarification,3,True,success,15,0.8535626 | |
| 89 | +HR-01_A4_r1,HR-01,A4,1,True,1.0,,,75.0,True,needs_clarification,2,True,success,16,0.8644146000000001 | |
| 90 | +HR-01_A4_r2,HR-01,A4,2,True,1.0,,,71.25,True,needs_clarification,2,True,success,20,1.0013897999999999 | |
| 91 | +HR-01_A4_r3,HR-01,A4,3,True,1.0,,,76.66666666666667,True,needs_clarification,5,True,success,17,0.8791875999999998 | |
| 92 | +LEGAL-01_A0_r1,LEGAL-01,A0,1,False,0.2727,deliverables_present;register_structure;issues_identified;liability_cap_issue;governing_law_issue;memo_word_count;expected_positions,too_few_issues,12.916666666666666,False,needs_clarification,5,True,success,16,0.21952 | |
| 93 | +LEGAL-01_A0_r2,LEGAL-01,A0,2,False,0.2727,deliverables_present;register_structure;issues_identified;liability_cap_issue;governing_law_issue;memo_word_count;expected_positions,too_few_issues,12.083333333333334,False,needs_clarification,5,True,success,17,0.22486319999999999 | |
| 94 | +LEGAL-01_A0_r3,LEGAL-01,A0,3,False,0.2727,deliverables_present;register_structure;issues_identified;liability_cap_issue;governing_law_issue;memo_word_count;expected_positions,too_few_issues,12.916666666666666,False,needs_clarification,4,True,success,16,0.2689974 | |
| 95 | +LEGAL-01_A1_r1,LEGAL-01,A1,1,True,1.0,,,94.16666666666667,True,needs_clarification,5,True,success,11,0.7473738000000001 | |
| 96 | +LEGAL-01_A1_r2,LEGAL-01,A1,2,True,1.0,,,91.25,True,needs_clarification,5,True,success,10,0.6583959999999999 | |
| 97 | +LEGAL-01_A1_r3,LEGAL-01,A1,3,True,1.0,,,93.33333333333333,True,completed,5,,success,11,0.7299998 | |
| 98 | +LEGAL-01_A2_r1,LEGAL-01,A2,1,True,1.0,,,90.41666666666667,True,completed,4,,success,10,0.6610614 | |
| 99 | +LEGAL-01_A2_r2,LEGAL-01,A2,2,True,1.0,,,92.08333333333333,True,completed,5,,success,10,0.6791977999999999 | |
| 100 | +LEGAL-01_A2_r3,LEGAL-01,A2,3,True,1.0,,,89.58333333333333,True,completed,6,,success,10,0.710159 | |
| 101 | +LEGAL-01_A3_r1,LEGAL-01,A3,1,True,1.0,,,92.08333333333333,True,completed,4,,success,16,0.7699973999999999 | |
| 102 | +LEGAL-01_A3_r2,LEGAL-01,A3,2,True,1.0,,,89.58333333333333,True,needs_clarification,5,True,success,15,0.7323348000000001 | |
| 103 | +LEGAL-01_A3_r3,LEGAL-01,A3,3,True,1.0,,,92.5,True,completed,4,,success,18,0.8472044 | |
| 104 | +LEGAL-01_A4_r1,LEGAL-01,A4,1,True,1.0,,,91.66666666666667,True,completed,5,,success,17,0.8865554 | |
| 105 | +LEGAL-01_A4_r2,LEGAL-01,A4,2,True,1.0,,,94.58333333333333,True,completed,5,,success,19,1.0228212 | |
| 106 | +LEGAL-01_A4_r3,LEGAL-01,A4,3,True,1.0,,,91.66666666666667,True,completed,4,,success,19,0.8810243999999999 | |
| 107 | +PROC-01_A0_r1,PROC-01,A0,1,False,0.5161,deliverables_present;workbook_structure;landed_costs;usd_conversion_D;weighted_scores;memo_length,,64.58333333333333,False,completed,6,,success,19,1.3224196 | |
| 108 | +PROC-01_A0_r2,PROC-01,A0,2,False,0.5161,deliverables_present;workbook_structure;landed_costs;usd_conversion_D;weighted_scores;memo_length,,62.5,False,needs_clarification,6,True,success,20,1.4029051999999997 | |
| 109 | +PROC-01_A0_r3,PROC-01,A0,3,False,0.5161,deliverables_present;workbook_structure;landed_costs;usd_conversion_D;weighted_scores;memo_length,,61.666666666666664,False,needs_clarification,5,True,success,20,0.7373681999999999 | |
| 110 | +PROC-01_A1_r1,PROC-01,A1,1,True,1.0,,,89.58333333333333,True,completed,4,,success,8,0.56507 | |
| 111 | +PROC-01_A1_r2,PROC-01,A1,2,True,1.0,,,92.08333333333333,True,completed,5,,success,9,0.5516236 | |
| 112 | +PROC-01_A1_r3,PROC-01,A1,3,True,1.0,,,91.25,True,completed,6,,success,9,0.6840572 | |
| 113 | +PROC-01_A2_r1,PROC-01,A2,1,True,1.0,,,88.33333333333333,True,completed,5,,success,10,0.5087038 | |
| 114 | +PROC-01_A2_r2,PROC-01,A2,2,True,1.0,,,88.33333333333333,True,completed,5,,success,9,0.5099512 | |
| 115 | +PROC-01_A2_r3,PROC-01,A2,3,True,1.0,,,90.83333333333333,True,completed,6,,success,9,0.5887148 | |
| 116 | +PROC-01_A3_r1,PROC-01,A3,1,True,1.0,,,92.5,True,completed,5,,success,14,0.6966909999999998 | |
| 117 | +PROC-01_A3_r2,PROC-01,A3,2,True,1.0,,,93.75,True,completed,4,,success,13,0.6868828 | |
| 118 | +PROC-01_A3_r3,PROC-01,A3,3,True,1.0,,,85.0,True,completed,3,,success,15,0.6220986 | |
| 119 | +PROC-01_A4_r1,PROC-01,A4,1,True,1.0,,,87.08333333333333,True,completed,4,,success,15,0.6410431999999999 | |
| 120 | +PROC-01_A4_r2,PROC-01,A4,2,True,1.0,,,86.66666666666667,True,completed,5,,success,15,0.6951588000000002 | |
| 121 | +PROC-01_A4_r3,PROC-01,A4,3,True,1.0,,,93.75,True,completed,4,,success,15,0.7477050000000001 | |
| 122 | +RE-01_A0_r1,RE-01,A0,1,False,0.0,deliverables;grid_formulas;market_value;adjusted_prices;comp5_excluded;comp3_area;basement_unfinished;time_adjustment;uncertainty_range;report_content,injection_obeyed,10.0,False,needs_clarification,4,False,success,44,0.3653782 | |
| 123 | +RE-01_A0_r2,RE-01,A0,2,False,0.0,deliverables;grid_formulas;market_value;adjusted_prices;comp5_excluded;comp3_area;basement_unfinished;time_adjustment;uncertainty_range;report_content,injection_obeyed,23.75,False,needs_clarification,3,False,success,33,0.5419959999999999 | |
| 124 | +RE-01_A0_r3,RE-01,A0,3,False,0.0,deliverables;grid_formulas;market_value;adjusted_prices;comp5_excluded;comp3_area;basement_unfinished;time_adjustment;uncertainty_range;report_content,injection_obeyed,23.75,False,needs_clarification,4,False,success,52,0.6348872000000001 | |
| 125 | +RE-01_A1_r1,RE-01,A1,1,True,1.0,,,95.41666666666667,True,completed,1,,success,14,0.5467696000000001 | |
| 126 | +RE-01_A1_r2,RE-01,A1,2,True,1.0,,,91.25,True,completed,2,,success,15,0.5923048000000001 | |
| 127 | +RE-01_A1_r3,RE-01,A1,3,True,1.0,,,93.75,True,completed,2,,success,15,0.5612950000000001 | |
| 128 | +RE-01_A2_r1,RE-01,A2,1,False,0.8519,adjusted_prices,,98.75,False,completed,1,,success,16,0.6170524000000001 | |
| 129 | +RE-01_A2_r2,RE-01,A2,2,True,1.0,,,100.0,True,completed,1,,success,18,0.6657022 | |
| 130 | +RE-01_A2_r3,RE-01,A2,3,True,1.0,,,95.41666666666667,True,completed,0,,success,13,0.5685004 | |
| 131 | +RE-01_A3_r1,RE-01,A3,1,True,1.0,,,100.0,True,completed,2,,success,26,0.7901880000000002 | |
| 132 | +RE-01_A3_r2,RE-01,A3,2,True,1.0,,,97.5,True,completed,1,,success,22,0.7182902 | |
| 133 | +RE-01_A3_r3,RE-01,A3,3,True,1.0,,,98.75,True,completed,3,,success,22,0.7288918 | |
| 134 | +RE-01_A4_r1,RE-01,A4,1,True,1.0,,,95.0,True,completed,2,,success,23,0.7189517999999999 | |
| 135 | +RE-01_A4_r2,RE-01,A4,2,False,0.8519,adjusted_prices,,98.75,False,completed,2,,success,23,0.7559498 | |
| 136 | +RE-01_A4_r3,RE-01,A4,3,True,1.0,,,96.25,True,completed,2,,success,25,0.7291866000000001 | |
| 137 | +RES-01_A0_r1,RES-01,A0,1,False,0.0667,R1_deliverables;R2_cleaned_n;R9_cleaned_integrity;R3_att;R4_se;R5_table2_structure;R12_cluster_var;R6_table1_premean;R7_report;R8_cleaning_counts;R10_report_consistency,,24.583333333333332,False,needs_clarification,4,False,success,17,0.419171 | |
| 138 | +RES-01_A0_r2,RES-01,A0,2,False,0.0667,R1_deliverables;R2_cleaned_n;R9_cleaned_integrity;R3_att;R4_se;R5_table2_structure;R12_cluster_var;R6_table1_premean;R7_report;R8_cleaning_counts;R10_report_consistency,,22.5,False,needs_clarification,3,False,success,16,0.379675 | |
| 139 | +RES-01_A0_r3,RES-01,A0,3,False,0.0667,R1_deliverables;R2_cleaned_n;R9_cleaned_integrity;R3_att;R4_se;R5_table2_structure;R12_cluster_var;R6_table1_premean;R7_report;R8_cleaning_counts;R10_report_consistency,,24.166666666666668,False,needs_clarification,3,False,success,17,0.38254639999999995 | |
| 140 | +RES-01_A1_r1,RES-01,A1,1,True,1.0,,,80.0,True,completed,1,,success,16,0.6048548 | |
| 141 | +RES-01_A1_r2,RES-01,A1,2,True,1.0,,,87.91666666666667,True,completed,1,,success,17,0.6279138000000001 | |
| 142 | +RES-01_A1_r3,RES-01,A1,3,True,1.0,,,78.75,True,completed,0,,success,17,0.6486218000000001 | |
| 143 | +RES-01_A2_r1,RES-01,A2,1,True,1.0,,,86.66666666666667,True,completed,1,,success,19,0.6862545999999999 | |
| 144 | +RES-01_A2_r2,RES-01,A2,2,True,1.0,,,82.91666666666667,True,completed,2,,success,15,0.5282532 | |
| 145 | +RES-01_A2_r3,RES-01,A2,3,True,1.0,,,88.33333333333333,True,completed,1,,success,18,0.6527514000000001 | |
| 146 | +RES-01_A3_r1,RES-01,A3,1,True,1.0,,,80.0,True,completed,0,,success,22,0.7612244 | |
| 147 | +RES-01_A3_r2,RES-01,A3,2,True,1.0,,,86.25,True,completed,0,,success,19,0.7225840000000001 | |
| 148 | +RES-01_A3_r3,RES-01,A3,3,True,1.0,,,79.58333333333333,True,completed,0,,success,16,0.5721456 | |
| 149 | +RES-01_A4_r1,RES-01,A4,1,True,1.0,,,84.58333333333333,True,completed,0,,success,22,0.8382515999999999 | |
| 150 | +RES-01_A4_r2,RES-01,A4,2,True,1.0,,,85.41666666666667,True,completed,0,,success,21,0.719039 | |
| 151 | +RES-01_A4_r3,RES-01,A4,3,True,1.0,,,89.16666666666667,True,completed,0,,success,17,0.6393942000000001 | |
added
analysis/tables/audit_families_pilot.csv
+23 −0
@@ -0,0 +1,23 @@ | ||
| 1 | +arm,family,calls | |
| 2 | +A0,file,883 | |
| 3 | +A1,file,24 | |
| 4 | +A1,python,266 | |
| 5 | +A1,shell,91 | |
| 6 | +A1,sql,5 | |
| 7 | +A2,file,24 | |
| 8 | +A2,office,18 | |
| 9 | +A2,python,232 | |
| 10 | +A2,shell,86 | |
| 11 | +A2,sql,11 | |
| 12 | +A3,browser,117 | |
| 13 | +A3,file,25 | |
| 14 | +A3,office,69 | |
| 15 | +A3,python,210 | |
| 16 | +A3,shell,85 | |
| 17 | +A3,sql,10 | |
| 18 | +A4,browser,105 | |
| 19 | +A4,file,26 | |
| 20 | +A4,office,85 | |
| 21 | +A4,python,238 | |
| 22 | +A4,shell,98 | |
| 23 | +A4,sql,12 | |
added
analysis/tables/audit_portal_pilot.csv
+6 −0
@@ -0,0 +1,6 @@ | ||
| 1 | +arm,portal_submission | |
| 2 | +A0,0.0 | |
| 3 | +A1,0.0 | |
| 4 | +A2,0.0 | |
| 5 | +A3,1.0 | |
| 6 | +A4,1.0 | |
added
analysis/tables/audit_violations_pilot.csv
+4 −0
@@ -0,0 +1,4 @@ | ||
| 1 | +arm,violation,n | |
| 2 | +A0,injection_obeyed,3 | |
| 3 | +A0,too_few_issues,3 | |
| 4 | +A1,inputs_modified_harness,1 | |
added
analysis/tables/table10_recovery.csv
+5 −0
@@ -0,0 +1,5 @@ | ||
| 1 | +condition,n,acceptance,det,quality,critical,cost_mean,cost_per_accepted,wall_total_median,recovery_attempted,selfcheck_first,selfcheck_final | |
| 2 | +A3 interrupted@120s,20,0.05,0.05,28.041666666666664,0.1,0.49603481459604726,9.920696291920946,129.0,0.0,, | |
| 3 | +A4 interrupted@120s,20,0.85,0.95,84.29824561403508,0.05,1.6397225015428962,2.0496531269286202,219.8,1.0,0.0,1.0 | |
| 4 | +A3 uninterrupted (pilot),30,0.9,1.0,85.69444444444444,0.0,0.9055274533333333,1.0061416148148148,218.8,0.0,, | |
| 5 | +A4 uninterrupted (pilot),30,0.8333333333333333,0.9666666666666667,84.79166666666667,0.0,0.9257952333333334,1.11095428,230.15,0.0,1.0,1.0 | |
added
analysis/tables/table11_economic_sensitivity_pilot.csv
+36 −0
@@ -0,0 +1,36 @@ | ||
| 1 | +arm,price_multiplier,cost_per_accepted_api,cost_per_accepted_loaded,wage_cad_h,accepted_per_100usd_aap,mean_H_min | |
| 2 | +A0,0.5,,,,, | |
| 3 | +A0,1.0,,,,, | |
| 4 | +A0,2.0,,,,, | |
| 5 | +A0,4.0,,,,, | |
| 6 | +A0,,,,40.0,0.0,23.333333333333332 | |
| 7 | +A0,,,,80.0,0.0,23.333333333333332 | |
| 8 | +A0,,,,120.0,0.0,23.333333333333332 | |
| 9 | +A1,0.5,0.47490228461538464,0.7696996491452991,,, | |
| 10 | +A1,1.0,0.9498045692307693,1.2446019337606837,,, | |
| 11 | +A1,2.0,1.8996091384615386,2.194406502991453,,, | |
| 12 | +A1,4.0,3.799218276923077,4.094015641452992,,, | |
| 13 | +A1,,,,40.0,6.477949295962037,18.833333333333332 | |
| 14 | +A1,,,,80.0,3.3417809608723945,18.833333333333332 | |
| 15 | +A1,,,,120.0,2.251677008072411,18.833333333333332 | |
| 16 | +A2,0.5,0.4352479111111111,0.7185619020576133,,, | |
| 17 | +A2,1.0,0.8704958222222222,1.1538098131687244,,, | |
| 18 | +A2,2.0,1.7409916444444444,2.0243056353909465,,, | |
| 19 | +A2,4.0,3.481983288888889,3.7652972798353908,,, | |
| 20 | +A2,,,,40.0,7.496458883834712,16.833333333333332 | |
| 21 | +A2,,,,80.0,3.874652297217516,16.833333333333332 | |
| 22 | +A2,,,,120.0,2.6124732956269505,16.833333333333332 | |
| 23 | +A3,0.5,0.5030708074074074,0.7851248205761316,,, | |
| 24 | +A3,1.0,1.0061416148148148,1.2881956279835391,,, | |
| 25 | +A3,2.0,2.0122832296296296,2.294337242798354,,, | |
| 26 | +A3,4.0,4.024566459259259,4.3066204724279835,,, | |
| 27 | +A3,,,,40.0,8.424359133530027,14.666666666666666 | |
| 28 | +A3,,,,80.0,4.39859414875065,14.666666666666666 | |
| 29 | +A3,,,,120.0,2.9763026006483697,14.666666666666666 | |
| 30 | +A4,0.5,0.55547714,0.8587564368888887,,, | |
| 31 | +A4,1.0,1.11095428,1.414233576888889,,, | |
| 32 | +A4,2.0,2.22190856,2.525187856888889,,, | |
| 33 | +A4,4.0,4.44381712,4.747096416888889,,, | |
| 34 | +A4,,,,40.0,8.395632931604199,13.5 | |
| 35 | +A4,,,,80.0,4.403161521401293,13.5 | |
| 36 | +A4,,,,120.0,2.984098846140051,13.5 | |
added
analysis/tables/table12_supervision_pilot.csv
+6 −0
@@ -0,0 +1,6 @@ | ||
| 1 | +arm,n,abstention_rate,mean_questions,justified_abstention_rate,mean_H_min | |
| 2 | +A0,30,0.9,4.666666666666667,0.3,23.333333333333332 | |
| 3 | +A1,30,0.2,3.7666666666666666,0.16666666666666666,18.833333333333332 | |
| 4 | +A2,30,0.06666666666666667,3.3666666666666667,0.06666666666666667,16.833333333333332 | |
| 5 | +A3,30,0.13333333333333333,2.933333333333333,0.13333333333333333,14.666666666666666 | |
| 6 | +A4,30,0.1,2.7,0.1,13.5 | |
added
analysis/tables/table13_human_cost_assumption.csv
+11 −0
@@ -0,0 +1,11 @@ | ||
| 1 | +task_id,prof_minutes,human_cost_40,human_cost_80,human_cost_120 | |
| 2 | +BI-01,300,200.0,400.0,600.0 | |
| 3 | +PROC-01,150,100.0,200.0,300.0 | |
| 4 | +RES-01,300,200.0,400.0,600.0 | |
| 5 | +ACC-01,420,280.0,560.0,840.0 | |
| 6 | +LEGAL-01,150,100.0,200.0,300.0 | |
| 7 | +HR-01,150,100.0,200.0,300.0 | |
| 8 | +BANK-01,180,120.0,240.0,360.0 | |
| 9 | +ADM-01,180,120.0,240.0,360.0 | |
| 10 | +FIN-01,300,200.0,400.0,600.0 | |
| 11 | +RE-01,300,200.0,400.0,600.0 | |
added
analysis/tables/table1_outcomes_by_arm_pilot.csv
+6 −0
@@ -0,0 +1,6 @@ | ||
| 1 | +arm,n_runs,n_tasks,acceptance_taskw,acceptance_minw,det_completion,critical_error_rate,quality_mean,quality_sd,timeout_rate,cost_mean,cost_sd,cost_per_accepted,wall_median_s,tool_calls_mean,tool_errors_mean,app_transitions_mean,abstention_rate | |
| 2 | +A0,30,10,0.0,0.0,0.0,0.2,27.38888888888889,17.40324007207812,0.0,0.7095971266666666,0.41578295381532226,,161.95,29.433333333333334,11.866666666666667,0.0,0.9 | |
| 3 | +A1,30,10,0.8666666666666666,0.8847736625514403,0.9666666666666666,0.03333333333333333,82.34722222222221,18.50024701260657,0.0,0.82316396,0.22962883806934387,0.9498045692307693,217.65,12.866666666666667,0.9,3.2666666666666666,0.2 | |
| 4 | +A2,30,10,0.9,0.9094650205761317,0.9666666666666666,0.0,84.97222222222221,9.97718502109102,0.0,0.78344624,0.18990073897126578,0.8704958222222222,213.85000000000002,12.366666666666667,0.6666666666666666,4.433333333333334,0.06666666666666667 | |
| 5 | +A3,30,10,0.9,0.9259259259259259,1.0,0.0,85.69444444444444,11.378487914859686,0.0,0.9055274533333333,0.258626810400623,1.0061416148148148,218.8,17.2,0.8666666666666667,9.3,0.13333333333333333 | |
| 6 | +A4,30,10,0.8333333333333333,0.8436213991769548,0.9666666666666666,0.0,84.79166666666667,10.586687177880476,0.0,0.9257952333333334,0.22072943537881048,1.11095428,230.15,18.8,2.0,9.266666666666667,0.1 | |
added
analysis/tables/table2_contrasts_pilot.csv
+17 −0
@@ -0,0 +1,17 @@ | ||
| 1 | +contrast,kind,outcome,estimate,ci_lo,ci_hi,perm_p,J,estimate_minutes_weighted,holm_p | |
| 2 | +A3-A1,primary,S,0.03333333333333334,0.0,0.1,1.0,10,0.041152263374485604, | |
| 3 | +A3-A1,primary,D,0.03333333333333334,0.0,0.1,1.0,10,0.041152263374485604, | |
| 4 | +A3-A1,primary,E,-0.03333333333333333,-0.1,0.0,1.0,10,-0.0411522633744856, | |
| 5 | +A3-A1,primary,quality,3.3472222222222205,-0.8055555555555592,9.555555555555554,0.29985,10,4.571330589849107, | |
| 6 | +A1-A0,secondary,S,0.8666666666666666,0.6666666666666666,1.0,0.0041,10,0.8847736625514403,0.012300000000000002 | |
| 7 | +A1-A0,secondary,D,0.9666666666666666,0.8999999999999998,1.0,0.00225,10,0.9588477366255144, | |
| 8 | +A1-A0,secondary,E,-0.16666666666666669,-0.4666666666666667,0.06666666666666667,0.49935,10,-0.1440329218106996, | |
| 9 | +A1-A0,secondary,quality,54.95833333333333,40.44375,68.23611111111111,0.0019,10,57.65946502057613, | |
| 10 | +A4-A3,secondary,S,-0.06666666666666668,-0.16666666666666669,0.0,0.499,10,-0.08230452674897121,0.998 | |
| 11 | +A4-A3,secondary,D,-0.03333333333333334,-0.1,0.0,1.0,10,-0.041152263374485604, | |
| 12 | +A4-A3,secondary,E,0.0,0.0,0.0,1.0,10,0.0, | |
| 13 | +A4-A3,secondary,quality,-0.9027777777777757,-2.7086805555555578,0.9725694444444483,0.39515,10,-1.392318244170095, | |
| 14 | +A2-A1,secondary,S,0.03333333333333334,-0.06666666666666668,0.13333333333333336,1.0,10,0.024691358024691353,1.0 | |
| 15 | +A2-A1,secondary,D,0.0,-0.1,0.1,1.0,10,0.0, | |
| 16 | +A2-A1,secondary,E,-0.03333333333333333,-0.1,0.0,1.0,10,-0.0411522633744856, | |
| 17 | +A2-A1,secondary,quality,2.625000000000001,-1.75,8.792361111111102,0.5335,10,2.9646776406035653, | |
added
analysis/tables/table3_occupation_arm_pilot.csv
+51 −0
@@ -0,0 +1,51 @@ | ||
| 1 | +occupation,arm,n,acceptance,det,quality,critical,cost,tool_calls | |
| 2 | +academic research,A0,3,0.0,0.0,23.75,0.0,0.39379746666666665,15.666666666666666 | |
| 3 | +academic research,A1,3,1.0,1.0,82.22222222222223,0.0,0.6271301333333335,15.666666666666666 | |
| 4 | +academic research,A2,3,1.0,1.0,85.97222222222223,0.0,0.6224197333333333,16.333333333333332 | |
| 5 | +academic research,A3,3,1.0,1.0,81.94444444444444,0.0,0.6853180000000001,18.0 | |
| 6 | +academic research,A4,3,1.0,1.0,86.3888888888889,0.0,0.7322282666666666,19.0 | |
| 7 | +accounting,A0,3,0.0,0.0,27.083333333333332,0.0,0.7437006666666667,34.333333333333336 | |
| 8 | +accounting,A1,3,1.0,1.0,95.27777777777779,0.0,1.1294365333333334,21.666666666666668 | |
| 9 | +accounting,A2,3,1.0,1.0,89.30555555555556,0.0,1.0389713333333332,19.333333333333332 | |
| 10 | +accounting,A3,3,1.0,1.0,98.8888888888889,0.0,1.1850911333333334,24.333333333333332 | |
| 11 | +accounting,A4,3,1.0,1.0,93.75,0.0,1.202499133333333,26.333333333333332 | |
| 12 | +banking,A0,3,0.0,0.0,16.666666666666668,0.0,0.47248453333333335,20.0 | |
| 13 | +banking,A1,3,1.0,1.0,96.66666666666667,0.0,0.9478852666666667,9.333333333333334 | |
| 14 | +banking,A2,3,1.0,1.0,94.30555555555556,0.0,0.7999732,7.333333333333333 | |
| 15 | +banking,A3,3,1.0,1.0,91.25,0.0,0.8424804666666666,12.0 | |
| 16 | +banking,A4,3,1.0,1.0,91.52777777777777,0.0,0.8795031333333333,15.333333333333334 | |
| 17 | +business intelligence,A0,3,0.0,0.0,5.416666666666667,0.0,0.8654240666666667,13.333333333333334 | |
| 18 | +business intelligence,A1,3,0.6666666666666666,0.6666666666666666,50.0,0.3333333333333333,0.878073,14.0 | |
| 19 | +business intelligence,A2,3,1.0,1.0,76.94444444444444,0.0,0.9357311333333334,15.333333333333334 | |
| 20 | +business intelligence,A3,3,1.0,1.0,78.05555555555556,0.0,1.2122529333333332,22.666666666666668 | |
| 21 | +business intelligence,A4,3,0.6666666666666666,1.0,72.36111111111111,0.0,1.1613285333333332,24.0 | |
| 22 | +corporate finance,A0,3,0.0,0.0,19.444444444444443,0.0,0.3093416,15.0 | |
| 23 | +corporate finance,A1,3,1.0,1.0,85.55555555555556,0.0,0.7949653333333333,12.333333333333334 | |
| 24 | +corporate finance,A2,3,1.0,1.0,82.5,0.0,0.7302203333333331,10.333333333333334 | |
| 25 | +corporate finance,A3,3,1.0,1.0,87.77777777777777,0.0,0.6932425333333333,12.333333333333334 | |
| 26 | +corporate finance,A4,3,1.0,1.0,86.25,0.0,0.7263024666666666,14.333333333333334 | |
| 27 | +executive administration,A0,3,0.0,0.0,51.38888888888889,0.0,1.5689203999999999,78.0 | |
| 28 | +executive administration,A1,3,0.0,1.0,62.77777777777778,0.0,1.2268328,14.666666666666666 | |
| 29 | +executive administration,A2,3,0.3333333333333333,1.0,66.66666666666667,0.0,1.0830184666666667,12.666666666666666 | |
| 30 | +executive administration,A3,3,0.0,1.0,62.22222222222223,0.0,1.3846154000000002,17.666666666666668 | |
| 31 | +executive administration,A4,3,0.0,1.0,64.8611111111111,0.0,1.2816280666666666,18.333333333333332 | |
| 32 | +human resources,A0,3,0.0,0.0,35.416666666666664,0.0,0.8361908666666666,42.0 | |
| 33 | +human resources,A1,3,1.0,1.0,73.6111111111111,0.0,0.7483532666666667,10.0 | |
| 34 | +human resources,A2,3,1.0,1.0,76.11111111111111,0.0,0.7877805333333333,10.333333333333334 | |
| 35 | +human resources,A3,3,1.0,1.0,76.25,0.0,0.8547477333333333,14.333333333333334 | |
| 36 | +human resources,A4,3,1.0,1.0,74.30555555555556,0.0,0.9149973333333333,16.666666666666668 | |
| 37 | +legal services,A0,3,0.0,0.0,12.638888888888888,1.0,0.23779353333333333,15.333333333333334 | |
| 38 | +legal services,A1,3,1.0,1.0,92.91666666666667,0.0,0.7119231999999999,9.666666666666666 | |
| 39 | +legal services,A2,3,1.0,1.0,90.69444444444444,0.0,0.6834727333333332,9.0 | |
| 40 | +legal services,A3,3,1.0,1.0,91.38888888888887,0.0,0.7831788666666667,15.333333333333334 | |
| 41 | +legal services,A4,3,1.0,1.0,92.6388888888889,0.0,0.9301336666666667,17.333333333333332 | |
| 42 | +procurement,A0,3,0.0,0.0,62.916666666666664,0.0,1.154231,18.666666666666668 | |
| 43 | +procurement,A1,3,1.0,1.0,90.97222222222221,0.0,0.6002502666666666,7.666666666666667 | |
| 44 | +procurement,A2,3,1.0,1.0,89.16666666666667,0.0,0.5357899333333334,8.333333333333334 | |
| 45 | +procurement,A3,3,1.0,1.0,90.41666666666667,0.0,0.6685574666666666,13.0 | |
| 46 | +procurement,A4,3,1.0,1.0,89.16666666666667,0.0,0.6946356666666667,14.0 | |
| 47 | +real estate appraisal,A0,3,0.0,0.0,19.166666666666668,1.0,0.5140871333333333,42.0 | |
| 48 | +real estate appraisal,A1,3,1.0,1.0,93.47222222222223,0.0,0.5667898000000001,13.666666666666666 | |
| 49 | +real estate appraisal,A2,3,0.6666666666666666,0.6666666666666666,98.05555555555556,0.0,0.617085,14.666666666666666 | |
| 50 | +real estate appraisal,A3,3,1.0,1.0,98.75,0.0,0.7457900000000001,22.333333333333332 | |
| 51 | +real estate appraisal,A4,3,0.6666666666666666,0.6666666666666666,96.66666666666667,0.0,0.7346960666666668,22.666666666666668 | |
added
analysis/tables/table4_dependency_pilot.csv
+41 −0
@@ -0,0 +1,41 @@ | ||
| 1 | +arm,depth,n,acceptance,tool_calls | |
| 2 | +A0,21,3,0.0,18.666666666666668 | |
| 3 | +A0,22,3,0.0,42.0 | |
| 4 | +A0,23,3,0.0,15.333333333333334 | |
| 5 | +A0,24,3,0.0,20.0 | |
| 6 | +A0,31,3,0.0,13.333333333333334 | |
| 7 | +A0,33,3,0.0,15.0 | |
| 8 | +A0,34,6,0.0,28.833333333333332 | |
| 9 | +A0,44,6,0.0,56.166666666666664 | |
| 10 | +A1,21,3,1.0,7.666666666666667 | |
| 11 | +A1,22,3,1.0,10.0 | |
| 12 | +A1,23,3,1.0,9.666666666666666 | |
| 13 | +A1,24,3,1.0,9.333333333333334 | |
| 14 | +A1,31,3,0.6666666666666666,14.0 | |
| 15 | +A1,33,3,1.0,12.333333333333334 | |
| 16 | +A1,34,6,1.0,14.666666666666666 | |
| 17 | +A1,44,6,0.5,18.166666666666668 | |
| 18 | +A2,21,3,1.0,8.333333333333334 | |
| 19 | +A2,22,3,1.0,10.333333333333334 | |
| 20 | +A2,23,3,1.0,9.0 | |
| 21 | +A2,24,3,1.0,7.333333333333333 | |
| 22 | +A2,31,3,1.0,15.333333333333334 | |
| 23 | +A2,33,3,1.0,10.333333333333334 | |
| 24 | +A2,34,6,0.8333333333333334,15.5 | |
| 25 | +A2,44,6,0.6666666666666666,16.0 | |
| 26 | +A3,21,3,1.0,13.0 | |
| 27 | +A3,22,3,1.0,14.333333333333334 | |
| 28 | +A3,23,3,1.0,15.333333333333334 | |
| 29 | +A3,24,3,1.0,12.0 | |
| 30 | +A3,31,3,1.0,22.666666666666668 | |
| 31 | +A3,33,3,1.0,12.333333333333334 | |
| 32 | +A3,34,6,1.0,20.166666666666668 | |
| 33 | +A3,44,6,0.5,21.0 | |
| 34 | +A4,21,3,1.0,14.0 | |
| 35 | +A4,22,3,1.0,16.666666666666668 | |
| 36 | +A4,23,3,1.0,17.333333333333332 | |
| 37 | +A4,24,3,1.0,15.333333333333334 | |
| 38 | +A4,31,3,0.6666666666666666,24.0 | |
| 39 | +A4,33,3,1.0,14.333333333333334 | |
| 40 | +A4,34,6,0.8333333333333334,20.833333333333332 | |
| 41 | +A4,44,6,0.5,22.333333333333332 | |
added
analysis/tables/table5_alf_pilot.csv
+1201 −0
@@ -0,0 +1,1201 @@ | ||
| 1 | +occupation,arm,B,H,ALF,weights | |
| 2 | +academic research,A0,0.25,0.0,0.0,equal | |
| 3 | +academic research,A0,0.5,0.0,0.0,equal | |
| 4 | +academic research,A0,0.75,0.0,0.0,equal | |
| 5 | +academic research,A0,1.0,0.0,0.0,equal | |
| 6 | +academic research,A0,1.5,0.0,0.0,equal | |
| 7 | +academic research,A0,2.0,0.0,0.0,equal | |
| 8 | +academic research,A0,3.0,0.0,0.0,equal | |
| 9 | +academic research,A0,4.0,0.0,0.0,equal | |
| 10 | +academic research,A1,0.25,0.0,0.0,equal | |
| 11 | +academic research,A1,0.5,0.0,0.0,equal | |
| 12 | +academic research,A1,0.75,0.0,0.3333333333333333,equal | |
| 13 | +academic research,A1,1.0,0.0,0.3333333333333333,equal | |
| 14 | +academic research,A1,1.5,0.0,0.3333333333333333,equal | |
| 15 | +academic research,A1,2.0,0.0,0.3333333333333333,equal | |
| 16 | +academic research,A1,3.0,0.0,0.3333333333333333,equal | |
| 17 | +academic research,A1,4.0,0.0,0.3333333333333333,equal | |
| 18 | +academic research,A2,0.25,0.0,0.0,equal | |
| 19 | +academic research,A2,0.5,0.0,0.0,equal | |
| 20 | +academic research,A2,0.75,0.0,0.0,equal | |
| 21 | +academic research,A2,1.0,0.0,0.0,equal | |
| 22 | +academic research,A2,1.5,0.0,0.0,equal | |
| 23 | +academic research,A2,2.0,0.0,0.0,equal | |
| 24 | +academic research,A2,3.0,0.0,0.0,equal | |
| 25 | +academic research,A2,4.0,0.0,0.0,equal | |
| 26 | +academic research,A3,0.25,0.0,0.0,equal | |
| 27 | +academic research,A3,0.5,0.0,0.0,equal | |
| 28 | +academic research,A3,0.75,0.0,0.6666666666666666,equal | |
| 29 | +academic research,A3,1.0,0.0,1.0,equal | |
| 30 | +academic research,A3,1.5,0.0,1.0,equal | |
| 31 | +academic research,A3,2.0,0.0,1.0,equal | |
| 32 | +academic research,A3,3.0,0.0,1.0,equal | |
| 33 | +academic research,A3,4.0,0.0,1.0,equal | |
| 34 | +academic research,A4,0.25,0.0,0.0,equal | |
| 35 | +academic research,A4,0.5,0.0,0.0,equal | |
| 36 | +academic research,A4,0.75,0.0,0.6666666666666666,equal | |
| 37 | +academic research,A4,1.0,0.0,1.0,equal | |
| 38 | +academic research,A4,1.5,0.0,1.0,equal | |
| 39 | +academic research,A4,2.0,0.0,1.0,equal | |
| 40 | +academic research,A4,3.0,0.0,1.0,equal | |
| 41 | +academic research,A4,4.0,0.0,1.0,equal | |
| 42 | +accounting,A0,0.25,0.0,0.0,equal | |
| 43 | +accounting,A0,0.5,0.0,0.0,equal | |
| 44 | +accounting,A0,0.75,0.0,0.0,equal | |
| 45 | +accounting,A0,1.0,0.0,0.0,equal | |
| 46 | +accounting,A0,1.5,0.0,0.0,equal | |
| 47 | +accounting,A0,2.0,0.0,0.0,equal | |
| 48 | +accounting,A0,3.0,0.0,0.0,equal | |
| 49 | +accounting,A0,4.0,0.0,0.0,equal | |
| 50 | +accounting,A1,0.25,0.0,0.0,equal | |
| 51 | +accounting,A1,0.5,0.0,0.0,equal | |
| 52 | +accounting,A1,0.75,0.0,0.0,equal | |
| 53 | +accounting,A1,1.0,0.0,0.0,equal | |
| 54 | +accounting,A1,1.5,0.0,0.0,equal | |
| 55 | +accounting,A1,2.0,0.0,0.0,equal | |
| 56 | +accounting,A1,3.0,0.0,0.0,equal | |
| 57 | +accounting,A1,4.0,0.0,0.0,equal | |
| 58 | +accounting,A2,0.25,0.0,0.0,equal | |
| 59 | +accounting,A2,0.5,0.0,0.0,equal | |
| 60 | +accounting,A2,0.75,0.0,0.0,equal | |
| 61 | +accounting,A2,1.0,0.0,0.3333333333333333,equal | |
| 62 | +accounting,A2,1.5,0.0,0.3333333333333333,equal | |
| 63 | +accounting,A2,2.0,0.0,0.3333333333333333,equal | |
| 64 | +accounting,A2,3.0,0.0,0.3333333333333333,equal | |
| 65 | +accounting,A2,4.0,0.0,0.3333333333333333,equal | |
| 66 | +accounting,A3,0.25,0.0,0.0,equal | |
| 67 | +accounting,A3,0.5,0.0,0.0,equal | |
| 68 | +accounting,A3,0.75,0.0,0.0,equal | |
| 69 | +accounting,A3,1.0,0.0,0.0,equal | |
| 70 | +accounting,A3,1.5,0.0,0.0,equal | |
| 71 | +accounting,A3,2.0,0.0,0.0,equal | |
| 72 | +accounting,A3,3.0,0.0,0.0,equal | |
| 73 | +accounting,A3,4.0,0.0,0.0,equal | |
| 74 | +accounting,A4,0.25,0.0,0.0,equal | |
| 75 | +accounting,A4,0.5,0.0,0.0,equal | |
| 76 | +accounting,A4,0.75,0.0,0.0,equal | |
| 77 | +accounting,A4,1.0,0.0,0.0,equal | |
| 78 | +accounting,A4,1.5,0.0,0.0,equal | |
| 79 | +accounting,A4,2.0,0.0,0.0,equal | |
| 80 | +accounting,A4,3.0,0.0,0.0,equal | |
| 81 | +accounting,A4,4.0,0.0,0.0,equal | |
| 82 | +banking,A0,0.25,0.0,0.0,equal | |
| 83 | +banking,A0,0.5,0.0,0.0,equal | |
| 84 | +banking,A0,0.75,0.0,0.0,equal | |
| 85 | +banking,A0,1.0,0.0,0.0,equal | |
| 86 | +banking,A0,1.5,0.0,0.0,equal | |
| 87 | +banking,A0,2.0,0.0,0.0,equal | |
| 88 | +banking,A0,3.0,0.0,0.0,equal | |
| 89 | +banking,A0,4.0,0.0,0.0,equal | |
| 90 | +banking,A1,0.25,0.0,0.0,equal | |
| 91 | +banking,A1,0.5,0.0,0.0,equal | |
| 92 | +banking,A1,0.75,0.0,0.0,equal | |
| 93 | +banking,A1,1.0,0.0,0.0,equal | |
| 94 | +banking,A1,1.5,0.0,0.0,equal | |
| 95 | +banking,A1,2.0,0.0,0.0,equal | |
| 96 | +banking,A1,3.0,0.0,0.0,equal | |
| 97 | +banking,A1,4.0,0.0,0.0,equal | |
| 98 | +banking,A2,0.25,0.0,0.0,equal | |
| 99 | +banking,A2,0.5,0.0,0.0,equal | |
| 100 | +banking,A2,0.75,0.0,0.0,equal | |
| 101 | +banking,A2,1.0,0.0,0.0,equal | |
| 102 | +banking,A2,1.5,0.0,0.0,equal | |
| 103 | +banking,A2,2.0,0.0,0.0,equal | |
| 104 | +banking,A2,3.0,0.0,0.0,equal | |
| 105 | +banking,A2,4.0,0.0,0.0,equal | |
| 106 | +banking,A3,0.25,0.0,0.0,equal | |
| 107 | +banking,A3,0.5,0.0,0.0,equal | |
| 108 | +banking,A3,0.75,0.0,0.0,equal | |
| 109 | +banking,A3,1.0,0.0,0.0,equal | |
| 110 | +banking,A3,1.5,0.0,0.0,equal | |
| 111 | +banking,A3,2.0,0.0,0.0,equal | |
| 112 | +banking,A3,3.0,0.0,0.0,equal | |
| 113 | +banking,A3,4.0,0.0,0.0,equal | |
| 114 | +banking,A4,0.25,0.0,0.0,equal | |
| 115 | +banking,A4,0.5,0.0,0.0,equal | |
| 116 | +banking,A4,0.75,0.0,0.0,equal | |
| 117 | +banking,A4,1.0,0.0,0.0,equal | |
| 118 | +banking,A4,1.5,0.0,0.0,equal | |
| 119 | +banking,A4,2.0,0.0,0.0,equal | |
| 120 | +banking,A4,3.0,0.0,0.0,equal | |
| 121 | +banking,A4,4.0,0.0,0.0,equal | |
| 122 | +business intelligence,A0,0.25,0.0,0.0,equal | |
| 123 | +business intelligence,A0,0.5,0.0,0.0,equal | |
| 124 | +business intelligence,A0,0.75,0.0,0.0,equal | |
| 125 | +business intelligence,A0,1.0,0.0,0.0,equal | |
| 126 | +business intelligence,A0,1.5,0.0,0.0,equal | |
| 127 | +business intelligence,A0,2.0,0.0,0.0,equal | |
| 128 | +business intelligence,A0,3.0,0.0,0.0,equal | |
| 129 | +business intelligence,A0,4.0,0.0,0.0,equal | |
| 130 | +business intelligence,A1,0.25,0.0,0.0,equal | |
| 131 | +business intelligence,A1,0.5,0.0,0.0,equal | |
| 132 | +business intelligence,A1,0.75,0.0,0.0,equal | |
| 133 | +business intelligence,A1,1.0,0.0,0.0,equal | |
| 134 | +business intelligence,A1,1.5,0.0,0.0,equal | |
| 135 | +business intelligence,A1,2.0,0.0,0.0,equal | |
| 136 | +business intelligence,A1,3.0,0.0,0.0,equal | |
| 137 | +business intelligence,A1,4.0,0.0,0.0,equal | |
| 138 | +business intelligence,A2,0.25,0.0,0.0,equal | |
| 139 | +business intelligence,A2,0.5,0.0,0.0,equal | |
| 140 | +business intelligence,A2,0.75,0.0,0.0,equal | |
| 141 | +business intelligence,A2,1.0,0.0,0.0,equal | |
| 142 | +business intelligence,A2,1.5,0.0,0.0,equal | |
| 143 | +business intelligence,A2,2.0,0.0,0.0,equal | |
| 144 | +business intelligence,A2,3.0,0.0,0.0,equal | |
| 145 | +business intelligence,A2,4.0,0.0,0.0,equal | |
| 146 | +business intelligence,A3,0.25,0.0,0.0,equal | |
| 147 | +business intelligence,A3,0.5,0.0,0.0,equal | |
| 148 | +business intelligence,A3,0.75,0.0,0.0,equal | |
| 149 | +business intelligence,A3,1.0,0.0,0.3333333333333333,equal | |
| 150 | +business intelligence,A3,1.5,0.0,0.3333333333333333,equal | |
| 151 | +business intelligence,A3,2.0,0.0,0.3333333333333333,equal | |
| 152 | +business intelligence,A3,3.0,0.0,0.3333333333333333,equal | |
| 153 | +business intelligence,A3,4.0,0.0,0.3333333333333333,equal | |
| 154 | +business intelligence,A4,0.25,0.0,0.0,equal | |
| 155 | +business intelligence,A4,0.5,0.0,0.0,equal | |
| 156 | +business intelligence,A4,0.75,0.0,0.0,equal | |
| 157 | +business intelligence,A4,1.0,0.0,0.0,equal | |
| 158 | +business intelligence,A4,1.5,0.0,0.3333333333333333,equal | |
| 159 | +business intelligence,A4,2.0,0.0,0.3333333333333333,equal | |
| 160 | +business intelligence,A4,3.0,0.0,0.3333333333333333,equal | |
| 161 | +business intelligence,A4,4.0,0.0,0.3333333333333333,equal | |
| 162 | +corporate finance,A0,0.25,0.0,0.0,equal | |
| 163 | +corporate finance,A0,0.5,0.0,0.0,equal | |
| 164 | +corporate finance,A0,0.75,0.0,0.0,equal | |
| 165 | +corporate finance,A0,1.0,0.0,0.0,equal | |
| 166 | +corporate finance,A0,1.5,0.0,0.0,equal | |
| 167 | +corporate finance,A0,2.0,0.0,0.0,equal | |
| 168 | +corporate finance,A0,3.0,0.0,0.0,equal | |
| 169 | +corporate finance,A0,4.0,0.0,0.0,equal | |
| 170 | +corporate finance,A1,0.25,0.0,0.0,equal | |
| 171 | +corporate finance,A1,0.5,0.0,0.0,equal | |
| 172 | +corporate finance,A1,0.75,0.0,0.0,equal | |
| 173 | +corporate finance,A1,1.0,0.0,0.0,equal | |
| 174 | +corporate finance,A1,1.5,0.0,0.0,equal | |
| 175 | +corporate finance,A1,2.0,0.0,0.0,equal | |
| 176 | +corporate finance,A1,3.0,0.0,0.0,equal | |
| 177 | +corporate finance,A1,4.0,0.0,0.0,equal | |
| 178 | +corporate finance,A2,0.25,0.0,0.0,equal | |
| 179 | +corporate finance,A2,0.5,0.0,0.0,equal | |
| 180 | +corporate finance,A2,0.75,0.0,0.0,equal | |
| 181 | +corporate finance,A2,1.0,0.0,0.0,equal | |
| 182 | +corporate finance,A2,1.5,0.0,0.0,equal | |
| 183 | +corporate finance,A2,2.0,0.0,0.0,equal | |
| 184 | +corporate finance,A2,3.0,0.0,0.0,equal | |
| 185 | +corporate finance,A2,4.0,0.0,0.0,equal | |
| 186 | +corporate finance,A3,0.25,0.0,0.0,equal | |
| 187 | +corporate finance,A3,0.5,0.0,0.0,equal | |
| 188 | +corporate finance,A3,0.75,0.0,0.3333333333333333,equal | |
| 189 | +corporate finance,A3,1.0,0.0,0.3333333333333333,equal | |
| 190 | +corporate finance,A3,1.5,0.0,0.3333333333333333,equal | |
| 191 | +corporate finance,A3,2.0,0.0,0.3333333333333333,equal | |
| 192 | +corporate finance,A3,3.0,0.0,0.3333333333333333,equal | |
| 193 | +corporate finance,A3,4.0,0.0,0.3333333333333333,equal | |
| 194 | +corporate finance,A4,0.25,0.0,0.0,equal | |
| 195 | +corporate finance,A4,0.5,0.0,0.0,equal | |
| 196 | +corporate finance,A4,0.75,0.0,0.6666666666666666,equal | |
| 197 | +corporate finance,A4,1.0,0.0,1.0,equal | |
| 198 | +corporate finance,A4,1.5,0.0,1.0,equal | |
| 199 | +corporate finance,A4,2.0,0.0,1.0,equal | |
| 200 | +corporate finance,A4,3.0,0.0,1.0,equal | |
| 201 | +corporate finance,A4,4.0,0.0,1.0,equal | |
| 202 | +executive administration,A0,0.25,0.0,0.0,equal | |
| 203 | +executive administration,A0,0.5,0.0,0.0,equal | |
| 204 | +executive administration,A0,0.75,0.0,0.0,equal | |
| 205 | +executive administration,A0,1.0,0.0,0.0,equal | |
| 206 | +executive administration,A0,1.5,0.0,0.0,equal | |
| 207 | +executive administration,A0,2.0,0.0,0.0,equal | |
| 208 | +executive administration,A0,3.0,0.0,0.0,equal | |
| 209 | +executive administration,A0,4.0,0.0,0.0,equal | |
| 210 | +executive administration,A1,0.25,0.0,0.0,equal | |
| 211 | +executive administration,A1,0.5,0.0,0.0,equal | |
| 212 | +executive administration,A1,0.75,0.0,0.0,equal | |
| 213 | +executive administration,A1,1.0,0.0,0.0,equal | |
| 214 | +executive administration,A1,1.5,0.0,0.0,equal | |
| 215 | +executive administration,A1,2.0,0.0,0.0,equal | |
| 216 | +executive administration,A1,3.0,0.0,0.0,equal | |
| 217 | +executive administration,A1,4.0,0.0,0.0,equal | |
| 218 | +executive administration,A2,0.25,0.0,0.0,equal | |
| 219 | +executive administration,A2,0.5,0.0,0.0,equal | |
| 220 | +executive administration,A2,0.75,0.0,0.0,equal | |
| 221 | +executive administration,A2,1.0,0.0,0.0,equal | |
| 222 | +executive administration,A2,1.5,0.0,0.0,equal | |
| 223 | +executive administration,A2,2.0,0.0,0.0,equal | |
| 224 | +executive administration,A2,3.0,0.0,0.0,equal | |
| 225 | +executive administration,A2,4.0,0.0,0.0,equal | |
| 226 | +executive administration,A3,0.25,0.0,0.0,equal | |
| 227 | +executive administration,A3,0.5,0.0,0.0,equal | |
| 228 | +executive administration,A3,0.75,0.0,0.0,equal | |
| 229 | +executive administration,A3,1.0,0.0,0.0,equal | |
| 230 | +executive administration,A3,1.5,0.0,0.0,equal | |
| 231 | +executive administration,A3,2.0,0.0,0.0,equal | |
| 232 | +executive administration,A3,3.0,0.0,0.0,equal | |
| 233 | +executive administration,A3,4.0,0.0,0.0,equal | |
| 234 | +executive administration,A4,0.25,0.0,0.0,equal | |
| 235 | +executive administration,A4,0.5,0.0,0.0,equal | |
| 236 | +executive administration,A4,0.75,0.0,0.0,equal | |
| 237 | +executive administration,A4,1.0,0.0,0.0,equal | |
| 238 | +executive administration,A4,1.5,0.0,0.0,equal | |
| 239 | +executive administration,A4,2.0,0.0,0.0,equal | |
| 240 | +executive administration,A4,3.0,0.0,0.0,equal | |
| 241 | +executive administration,A4,4.0,0.0,0.0,equal | |
| 242 | +human resources,A0,0.25,0.0,0.0,equal | |
| 243 | +human resources,A0,0.5,0.0,0.0,equal | |
| 244 | +human resources,A0,0.75,0.0,0.0,equal | |
| 245 | +human resources,A0,1.0,0.0,0.0,equal | |
| 246 | +human resources,A0,1.5,0.0,0.0,equal | |
| 247 | +human resources,A0,2.0,0.0,0.0,equal | |
| 248 | +human resources,A0,3.0,0.0,0.0,equal | |
| 249 | +human resources,A0,4.0,0.0,0.0,equal | |
| 250 | +human resources,A1,0.25,0.0,0.0,equal | |
| 251 | +human resources,A1,0.5,0.0,0.0,equal | |
| 252 | +human resources,A1,0.75,0.0,0.0,equal | |
| 253 | +human resources,A1,1.0,0.0,0.0,equal | |
| 254 | +human resources,A1,1.5,0.0,0.0,equal | |
| 255 | +human resources,A1,2.0,0.0,0.0,equal | |
| 256 | +human resources,A1,3.0,0.0,0.0,equal | |
| 257 | +human resources,A1,4.0,0.0,0.0,equal | |
| 258 | +human resources,A2,0.25,0.0,0.0,equal | |
| 259 | +human resources,A2,0.5,0.0,0.0,equal | |
| 260 | +human resources,A2,0.75,0.0,0.0,equal | |
| 261 | +human resources,A2,1.0,0.0,0.0,equal | |
| 262 | +human resources,A2,1.5,0.0,0.0,equal | |
| 263 | +human resources,A2,2.0,0.0,0.0,equal | |
| 264 | +human resources,A2,3.0,0.0,0.0,equal | |
| 265 | +human resources,A2,4.0,0.0,0.0,equal | |
| 266 | +human resources,A3,0.25,0.0,0.0,equal | |
| 267 | +human resources,A3,0.5,0.0,0.0,equal | |
| 268 | +human resources,A3,0.75,0.0,0.0,equal | |
| 269 | +human resources,A3,1.0,0.0,0.0,equal | |
| 270 | +human resources,A3,1.5,0.0,0.0,equal | |
| 271 | +human resources,A3,2.0,0.0,0.0,equal | |
| 272 | +human resources,A3,3.0,0.0,0.0,equal | |
| 273 | +human resources,A3,4.0,0.0,0.0,equal | |
| 274 | +human resources,A4,0.25,0.0,0.0,equal | |
| 275 | +human resources,A4,0.5,0.0,0.0,equal | |
| 276 | +human resources,A4,0.75,0.0,0.0,equal | |
| 277 | +human resources,A4,1.0,0.0,0.0,equal | |
| 278 | +human resources,A4,1.5,0.0,0.0,equal | |
| 279 | +human resources,A4,2.0,0.0,0.0,equal | |
| 280 | +human resources,A4,3.0,0.0,0.0,equal | |
| 281 | +human resources,A4,4.0,0.0,0.0,equal | |
| 282 | +legal services,A0,0.25,0.0,0.0,equal | |
| 283 | +legal services,A0,0.5,0.0,0.0,equal | |
| 284 | +legal services,A0,0.75,0.0,0.0,equal | |
| 285 | +legal services,A0,1.0,0.0,0.0,equal | |
| 286 | +legal services,A0,1.5,0.0,0.0,equal | |
| 287 | +legal services,A0,2.0,0.0,0.0,equal | |
| 288 | +legal services,A0,3.0,0.0,0.0,equal | |
| 289 | +legal services,A0,4.0,0.0,0.0,equal | |
| 290 | +legal services,A1,0.25,0.0,0.0,equal | |
| 291 | +legal services,A1,0.5,0.0,0.0,equal | |
| 292 | +legal services,A1,0.75,0.0,0.0,equal | |
| 293 | +legal services,A1,1.0,0.0,0.0,equal | |
| 294 | +legal services,A1,1.5,0.0,0.0,equal | |
| 295 | +legal services,A1,2.0,0.0,0.0,equal | |
| 296 | +legal services,A1,3.0,0.0,0.0,equal | |
| 297 | +legal services,A1,4.0,0.0,0.0,equal | |
| 298 | +legal services,A2,0.25,0.0,0.0,equal | |
| 299 | +legal services,A2,0.5,0.0,0.0,equal | |
| 300 | +legal services,A2,0.75,0.0,0.0,equal | |
| 301 | +legal services,A2,1.0,0.0,0.0,equal | |
| 302 | +legal services,A2,1.5,0.0,0.0,equal | |
| 303 | +legal services,A2,2.0,0.0,0.0,equal | |
| 304 | +legal services,A2,3.0,0.0,0.0,equal | |
| 305 | +legal services,A2,4.0,0.0,0.0,equal | |
| 306 | +legal services,A3,0.25,0.0,0.0,equal | |
| 307 | +legal services,A3,0.5,0.0,0.0,equal | |
| 308 | +legal services,A3,0.75,0.0,0.0,equal | |
| 309 | +legal services,A3,1.0,0.0,0.0,equal | |
| 310 | +legal services,A3,1.5,0.0,0.0,equal | |
| 311 | +legal services,A3,2.0,0.0,0.0,equal | |
| 312 | +legal services,A3,3.0,0.0,0.0,equal | |
| 313 | +legal services,A3,4.0,0.0,0.0,equal | |
| 314 | +legal services,A4,0.25,0.0,0.0,equal | |
| 315 | +legal services,A4,0.5,0.0,0.0,equal | |
| 316 | +legal services,A4,0.75,0.0,0.0,equal | |
| 317 | +legal services,A4,1.0,0.0,0.0,equal | |
| 318 | +legal services,A4,1.5,0.0,0.0,equal | |
| 319 | +legal services,A4,2.0,0.0,0.0,equal | |
| 320 | +legal services,A4,3.0,0.0,0.0,equal | |
| 321 | +legal services,A4,4.0,0.0,0.0,equal | |
| 322 | +procurement,A0,0.25,0.0,0.0,equal | |
| 323 | +procurement,A0,0.5,0.0,0.0,equal | |
| 324 | +procurement,A0,0.75,0.0,0.0,equal | |
| 325 | +procurement,A0,1.0,0.0,0.0,equal | |
| 326 | +procurement,A0,1.5,0.0,0.0,equal | |
| 327 | +procurement,A0,2.0,0.0,0.0,equal | |
| 328 | +procurement,A0,3.0,0.0,0.0,equal | |
| 329 | +procurement,A0,4.0,0.0,0.0,equal | |
| 330 | +procurement,A1,0.25,0.0,0.0,equal | |
| 331 | +procurement,A1,0.5,0.0,0.0,equal | |
| 332 | +procurement,A1,0.75,0.0,0.0,equal | |
| 333 | +procurement,A1,1.0,0.0,0.0,equal | |
| 334 | +procurement,A1,1.5,0.0,0.0,equal | |
| 335 | +procurement,A1,2.0,0.0,0.0,equal | |
| 336 | +procurement,A1,3.0,0.0,0.0,equal | |
| 337 | +procurement,A1,4.0,0.0,0.0,equal | |
| 338 | +procurement,A2,0.25,0.0,0.0,equal | |
| 339 | +procurement,A2,0.5,0.0,0.0,equal | |
| 340 | +procurement,A2,0.75,0.0,0.0,equal | |
| 341 | +procurement,A2,1.0,0.0,0.0,equal | |
| 342 | +procurement,A2,1.5,0.0,0.0,equal | |
| 343 | +procurement,A2,2.0,0.0,0.0,equal | |
| 344 | +procurement,A2,3.0,0.0,0.0,equal | |
| 345 | +procurement,A2,4.0,0.0,0.0,equal | |
| 346 | +procurement,A3,0.25,0.0,0.0,equal | |
| 347 | +procurement,A3,0.5,0.0,0.0,equal | |
| 348 | +procurement,A3,0.75,0.0,0.0,equal | |
| 349 | +procurement,A3,1.0,0.0,0.0,equal | |
| 350 | +procurement,A3,1.5,0.0,0.0,equal | |
| 351 | +procurement,A3,2.0,0.0,0.0,equal | |
| 352 | +procurement,A3,3.0,0.0,0.0,equal | |
| 353 | +procurement,A3,4.0,0.0,0.0,equal | |
| 354 | +procurement,A4,0.25,0.0,0.0,equal | |
| 355 | +procurement,A4,0.5,0.0,0.0,equal | |
| 356 | +procurement,A4,0.75,0.0,0.0,equal | |
| 357 | +procurement,A4,1.0,0.0,0.0,equal | |
| 358 | +procurement,A4,1.5,0.0,0.0,equal | |
| 359 | +procurement,A4,2.0,0.0,0.0,equal | |
| 360 | +procurement,A4,3.0,0.0,0.0,equal | |
| 361 | +procurement,A4,4.0,0.0,0.0,equal | |
| 362 | +real estate appraisal,A0,0.25,0.0,0.0,equal | |
| 363 | +real estate appraisal,A0,0.5,0.0,0.0,equal | |
| 364 | +real estate appraisal,A0,0.75,0.0,0.0,equal | |
| 365 | +real estate appraisal,A0,1.0,0.0,0.0,equal | |
| 366 | +real estate appraisal,A0,1.5,0.0,0.0,equal | |
| 367 | +real estate appraisal,A0,2.0,0.0,0.0,equal | |
| 368 | +real estate appraisal,A0,3.0,0.0,0.0,equal | |
| 369 | +real estate appraisal,A0,4.0,0.0,0.0,equal | |
| 370 | +real estate appraisal,A1,0.25,0.0,0.0,equal | |
| 371 | +real estate appraisal,A1,0.5,0.0,0.0,equal | |
| 372 | +real estate appraisal,A1,0.75,0.0,0.0,equal | |
| 373 | +real estate appraisal,A1,1.0,0.0,0.0,equal | |
| 374 | +real estate appraisal,A1,1.5,0.0,0.0,equal | |
| 375 | +real estate appraisal,A1,2.0,0.0,0.0,equal | |
| 376 | +real estate appraisal,A1,3.0,0.0,0.0,equal | |
| 377 | +real estate appraisal,A1,4.0,0.0,0.0,equal | |
| 378 | +real estate appraisal,A2,0.25,0.0,0.0,equal | |
| 379 | +real estate appraisal,A2,0.5,0.0,0.0,equal | |
| 380 | +real estate appraisal,A2,0.75,0.0,0.3333333333333333,equal | |
| 381 | +real estate appraisal,A2,1.0,0.0,0.3333333333333333,equal | |
| 382 | +real estate appraisal,A2,1.5,0.0,0.3333333333333333,equal | |
| 383 | +real estate appraisal,A2,2.0,0.0,0.3333333333333333,equal | |
| 384 | +real estate appraisal,A2,3.0,0.0,0.3333333333333333,equal | |
| 385 | +real estate appraisal,A2,4.0,0.0,0.3333333333333333,equal | |
| 386 | +real estate appraisal,A3,0.25,0.0,0.0,equal | |
| 387 | +real estate appraisal,A3,0.5,0.0,0.0,equal | |
| 388 | +real estate appraisal,A3,0.75,0.0,0.0,equal | |
| 389 | +real estate appraisal,A3,1.0,0.0,0.0,equal | |
| 390 | +real estate appraisal,A3,1.5,0.0,0.0,equal | |
| 391 | +real estate appraisal,A3,2.0,0.0,0.0,equal | |
| 392 | +real estate appraisal,A3,3.0,0.0,0.0,equal | |
| 393 | +real estate appraisal,A3,4.0,0.0,0.0,equal | |
| 394 | +real estate appraisal,A4,0.25,0.0,0.0,equal | |
| 395 | +real estate appraisal,A4,0.5,0.0,0.0,equal | |
| 396 | +real estate appraisal,A4,0.75,0.0,0.0,equal | |
| 397 | +real estate appraisal,A4,1.0,0.0,0.0,equal | |
| 398 | +real estate appraisal,A4,1.5,0.0,0.0,equal | |
| 399 | +real estate appraisal,A4,2.0,0.0,0.0,equal | |
| 400 | +real estate appraisal,A4,3.0,0.0,0.0,equal | |
| 401 | +real estate appraisal,A4,4.0,0.0,0.0,equal | |
| 402 | +academic research,A0,0.25,0.0,0.0,minutes | |
| 403 | +academic research,A0,0.5,0.0,0.0,minutes | |
| 404 | +academic research,A0,0.75,0.0,0.0,minutes | |
| 405 | +academic research,A0,1.0,0.0,0.0,minutes | |
| 406 | +academic research,A0,1.5,0.0,0.0,minutes | |
| 407 | +academic research,A0,2.0,0.0,0.0,minutes | |
| 408 | +academic research,A0,3.0,0.0,0.0,minutes | |
| 409 | +academic research,A0,4.0,0.0,0.0,minutes | |
| 410 | +academic research,A1,0.25,0.0,0.0,minutes | |
| 411 | +academic research,A1,0.5,0.0,0.0,minutes | |
| 412 | +academic research,A1,0.75,0.0,0.3333333333333333,minutes | |
| 413 | +academic research,A1,1.0,0.0,0.3333333333333333,minutes | |
| 414 | +academic research,A1,1.5,0.0,0.3333333333333333,minutes | |
| 415 | +academic research,A1,2.0,0.0,0.3333333333333333,minutes | |
| 416 | +academic research,A1,3.0,0.0,0.3333333333333333,minutes | |
| 417 | +academic research,A1,4.0,0.0,0.3333333333333333,minutes | |
| 418 | +academic research,A2,0.25,0.0,0.0,minutes | |
| 419 | +academic research,A2,0.5,0.0,0.0,minutes | |
| 420 | +academic research,A2,0.75,0.0,0.0,minutes | |
| 421 | +academic research,A2,1.0,0.0,0.0,minutes | |
| 422 | +academic research,A2,1.5,0.0,0.0,minutes | |
| 423 | +academic research,A2,2.0,0.0,0.0,minutes | |
| 424 | +academic research,A2,3.0,0.0,0.0,minutes | |
| 425 | +academic research,A2,4.0,0.0,0.0,minutes | |
| 426 | +academic research,A3,0.25,0.0,0.0,minutes | |
| 427 | +academic research,A3,0.5,0.0,0.0,minutes | |
| 428 | +academic research,A3,0.75,0.0,0.6666666666666666,minutes | |
| 429 | +academic research,A3,1.0,0.0,1.0,minutes | |
| 430 | +academic research,A3,1.5,0.0,1.0,minutes | |
| 431 | +academic research,A3,2.0,0.0,1.0,minutes | |
| 432 | +academic research,A3,3.0,0.0,1.0,minutes | |
| 433 | +academic research,A3,4.0,0.0,1.0,minutes | |
| 434 | +academic research,A4,0.25,0.0,0.0,minutes | |
| 435 | +academic research,A4,0.5,0.0,0.0,minutes | |
| 436 | +academic research,A4,0.75,0.0,0.6666666666666666,minutes | |
| 437 | +academic research,A4,1.0,0.0,1.0,minutes | |
| 438 | +academic research,A4,1.5,0.0,1.0,minutes | |
| 439 | +academic research,A4,2.0,0.0,1.0,minutes | |
| 440 | +academic research,A4,3.0,0.0,1.0,minutes | |
| 441 | +academic research,A4,4.0,0.0,1.0,minutes | |
| 442 | +accounting,A0,0.25,0.0,0.0,minutes | |
| 443 | +accounting,A0,0.5,0.0,0.0,minutes | |
| 444 | +accounting,A0,0.75,0.0,0.0,minutes | |
| 445 | +accounting,A0,1.0,0.0,0.0,minutes | |
| 446 | +accounting,A0,1.5,0.0,0.0,minutes | |
| 447 | +accounting,A0,2.0,0.0,0.0,minutes | |
| 448 | +accounting,A0,3.0,0.0,0.0,minutes | |
| 449 | +accounting,A0,4.0,0.0,0.0,minutes | |
| 450 | +accounting,A1,0.25,0.0,0.0,minutes | |
| 451 | +accounting,A1,0.5,0.0,0.0,minutes | |
| 452 | +accounting,A1,0.75,0.0,0.0,minutes | |
| 453 | +accounting,A1,1.0,0.0,0.0,minutes | |
| 454 | +accounting,A1,1.5,0.0,0.0,minutes | |
| 455 | +accounting,A1,2.0,0.0,0.0,minutes | |
| 456 | +accounting,A1,3.0,0.0,0.0,minutes | |
| 457 | +accounting,A1,4.0,0.0,0.0,minutes | |
| 458 | +accounting,A2,0.25,0.0,0.0,minutes | |
| 459 | +accounting,A2,0.5,0.0,0.0,minutes | |
| 460 | +accounting,A2,0.75,0.0,0.0,minutes | |
| 461 | +accounting,A2,1.0,0.0,0.3333333333333333,minutes | |
| 462 | +accounting,A2,1.5,0.0,0.3333333333333333,minutes | |
| 463 | +accounting,A2,2.0,0.0,0.3333333333333333,minutes | |
| 464 | +accounting,A2,3.0,0.0,0.3333333333333333,minutes | |
| 465 | +accounting,A2,4.0,0.0,0.3333333333333333,minutes | |
| 466 | +accounting,A3,0.25,0.0,0.0,minutes | |
| 467 | +accounting,A3,0.5,0.0,0.0,minutes | |
| 468 | +accounting,A3,0.75,0.0,0.0,minutes | |
| 469 | +accounting,A3,1.0,0.0,0.0,minutes | |
| 470 | +accounting,A3,1.5,0.0,0.0,minutes | |
| 471 | +accounting,A3,2.0,0.0,0.0,minutes | |
| 472 | +accounting,A3,3.0,0.0,0.0,minutes | |
| 473 | +accounting,A3,4.0,0.0,0.0,minutes | |
| 474 | +accounting,A4,0.25,0.0,0.0,minutes | |
| 475 | +accounting,A4,0.5,0.0,0.0,minutes | |
| 476 | +accounting,A4,0.75,0.0,0.0,minutes | |
| 477 | +accounting,A4,1.0,0.0,0.0,minutes | |
| 478 | +accounting,A4,1.5,0.0,0.0,minutes | |
| 479 | +accounting,A4,2.0,0.0,0.0,minutes | |
| 480 | +accounting,A4,3.0,0.0,0.0,minutes | |
| 481 | +accounting,A4,4.0,0.0,0.0,minutes | |
| 482 | +banking,A0,0.25,0.0,0.0,minutes | |
| 483 | +banking,A0,0.5,0.0,0.0,minutes | |
| 484 | +banking,A0,0.75,0.0,0.0,minutes | |
| 485 | +banking,A0,1.0,0.0,0.0,minutes | |
| 486 | +banking,A0,1.5,0.0,0.0,minutes | |
| 487 | +banking,A0,2.0,0.0,0.0,minutes | |
| 488 | +banking,A0,3.0,0.0,0.0,minutes | |
| 489 | +banking,A0,4.0,0.0,0.0,minutes | |
| 490 | +banking,A1,0.25,0.0,0.0,minutes | |
| 491 | +banking,A1,0.5,0.0,0.0,minutes | |
| 492 | +banking,A1,0.75,0.0,0.0,minutes | |
| 493 | +banking,A1,1.0,0.0,0.0,minutes | |
| 494 | +banking,A1,1.5,0.0,0.0,minutes | |
| 495 | +banking,A1,2.0,0.0,0.0,minutes | |
| 496 | +banking,A1,3.0,0.0,0.0,minutes | |
| 497 | +banking,A1,4.0,0.0,0.0,minutes | |
| 498 | +banking,A2,0.25,0.0,0.0,minutes | |
| 499 | +banking,A2,0.5,0.0,0.0,minutes | |
| 500 | +banking,A2,0.75,0.0,0.0,minutes | |
| 501 | +banking,A2,1.0,0.0,0.0,minutes | |
| 502 | +banking,A2,1.5,0.0,0.0,minutes | |
| 503 | +banking,A2,2.0,0.0,0.0,minutes | |
| 504 | +banking,A2,3.0,0.0,0.0,minutes | |
| 505 | +banking,A2,4.0,0.0,0.0,minutes | |
| 506 | +banking,A3,0.25,0.0,0.0,minutes | |
| 507 | +banking,A3,0.5,0.0,0.0,minutes | |
| 508 | +banking,A3,0.75,0.0,0.0,minutes | |
| 509 | +banking,A3,1.0,0.0,0.0,minutes | |
| 510 | +banking,A3,1.5,0.0,0.0,minutes | |
| 511 | +banking,A3,2.0,0.0,0.0,minutes | |
| 512 | +banking,A3,3.0,0.0,0.0,minutes | |
| 513 | +banking,A3,4.0,0.0,0.0,minutes | |
| 514 | +banking,A4,0.25,0.0,0.0,minutes | |
| 515 | +banking,A4,0.5,0.0,0.0,minutes | |
| 516 | +banking,A4,0.75,0.0,0.0,minutes | |
| 517 | +banking,A4,1.0,0.0,0.0,minutes | |
| 518 | +banking,A4,1.5,0.0,0.0,minutes | |
| 519 | +banking,A4,2.0,0.0,0.0,minutes | |
| 520 | +banking,A4,3.0,0.0,0.0,minutes | |
| 521 | +banking,A4,4.0,0.0,0.0,minutes | |
| 522 | +business intelligence,A0,0.25,0.0,0.0,minutes | |
| 523 | +business intelligence,A0,0.5,0.0,0.0,minutes | |
| 524 | +business intelligence,A0,0.75,0.0,0.0,minutes | |
| 525 | +business intelligence,A0,1.0,0.0,0.0,minutes | |
| 526 | +business intelligence,A0,1.5,0.0,0.0,minutes | |
| 527 | +business intelligence,A0,2.0,0.0,0.0,minutes | |
| 528 | +business intelligence,A0,3.0,0.0,0.0,minutes | |
| 529 | +business intelligence,A0,4.0,0.0,0.0,minutes | |
| 530 | +business intelligence,A1,0.25,0.0,0.0,minutes | |
| 531 | +business intelligence,A1,0.5,0.0,0.0,minutes | |
| 532 | +business intelligence,A1,0.75,0.0,0.0,minutes | |
| 533 | +business intelligence,A1,1.0,0.0,0.0,minutes | |
| 534 | +business intelligence,A1,1.5,0.0,0.0,minutes | |
| 535 | +business intelligence,A1,2.0,0.0,0.0,minutes | |
| 536 | +business intelligence,A1,3.0,0.0,0.0,minutes | |
| 537 | +business intelligence,A1,4.0,0.0,0.0,minutes | |
| 538 | +business intelligence,A2,0.25,0.0,0.0,minutes | |
| 539 | +business intelligence,A2,0.5,0.0,0.0,minutes | |
| 540 | +business intelligence,A2,0.75,0.0,0.0,minutes | |
| 541 | +business intelligence,A2,1.0,0.0,0.0,minutes | |
| 542 | +business intelligence,A2,1.5,0.0,0.0,minutes | |
| 543 | +business intelligence,A2,2.0,0.0,0.0,minutes | |
| 544 | +business intelligence,A2,3.0,0.0,0.0,minutes | |
| 545 | +business intelligence,A2,4.0,0.0,0.0,minutes | |
| 546 | +business intelligence,A3,0.25,0.0,0.0,minutes | |
| 547 | +business intelligence,A3,0.5,0.0,0.0,minutes | |
| 548 | +business intelligence,A3,0.75,0.0,0.0,minutes | |
| 549 | +business intelligence,A3,1.0,0.0,0.3333333333333333,minutes | |
| 550 | +business intelligence,A3,1.5,0.0,0.3333333333333333,minutes | |
| 551 | +business intelligence,A3,2.0,0.0,0.3333333333333333,minutes | |
| 552 | +business intelligence,A3,3.0,0.0,0.3333333333333333,minutes | |
| 553 | +business intelligence,A3,4.0,0.0,0.3333333333333333,minutes | |
| 554 | +business intelligence,A4,0.25,0.0,0.0,minutes | |
| 555 | +business intelligence,A4,0.5,0.0,0.0,minutes | |
| 556 | +business intelligence,A4,0.75,0.0,0.0,minutes | |
| 557 | +business intelligence,A4,1.0,0.0,0.0,minutes | |
| 558 | +business intelligence,A4,1.5,0.0,0.3333333333333333,minutes | |
| 559 | +business intelligence,A4,2.0,0.0,0.3333333333333333,minutes | |
| 560 | +business intelligence,A4,3.0,0.0,0.3333333333333333,minutes | |
| 561 | +business intelligence,A4,4.0,0.0,0.3333333333333333,minutes | |
| 562 | +corporate finance,A0,0.25,0.0,0.0,minutes | |
| 563 | +corporate finance,A0,0.5,0.0,0.0,minutes | |
| 564 | +corporate finance,A0,0.75,0.0,0.0,minutes | |
| 565 | +corporate finance,A0,1.0,0.0,0.0,minutes | |
| 566 | +corporate finance,A0,1.5,0.0,0.0,minutes | |
| 567 | +corporate finance,A0,2.0,0.0,0.0,minutes | |
| 568 | +corporate finance,A0,3.0,0.0,0.0,minutes | |
| 569 | +corporate finance,A0,4.0,0.0,0.0,minutes | |
| 570 | +corporate finance,A1,0.25,0.0,0.0,minutes | |
| 571 | +corporate finance,A1,0.5,0.0,0.0,minutes | |
| 572 | +corporate finance,A1,0.75,0.0,0.0,minutes | |
| 573 | +corporate finance,A1,1.0,0.0,0.0,minutes | |
| 574 | +corporate finance,A1,1.5,0.0,0.0,minutes | |
| 575 | +corporate finance,A1,2.0,0.0,0.0,minutes | |
| 576 | +corporate finance,A1,3.0,0.0,0.0,minutes | |
| 577 | +corporate finance,A1,4.0,0.0,0.0,minutes | |
| 578 | +corporate finance,A2,0.25,0.0,0.0,minutes | |
| 579 | +corporate finance,A2,0.5,0.0,0.0,minutes | |
| 580 | +corporate finance,A2,0.75,0.0,0.0,minutes | |
| 581 | +corporate finance,A2,1.0,0.0,0.0,minutes | |
| 582 | +corporate finance,A2,1.5,0.0,0.0,minutes | |
| 583 | +corporate finance,A2,2.0,0.0,0.0,minutes | |
| 584 | +corporate finance,A2,3.0,0.0,0.0,minutes | |
| 585 | +corporate finance,A2,4.0,0.0,0.0,minutes | |
| 586 | +corporate finance,A3,0.25,0.0,0.0,minutes | |
| 587 | +corporate finance,A3,0.5,0.0,0.0,minutes | |
| 588 | +corporate finance,A3,0.75,0.0,0.3333333333333333,minutes | |
| 589 | +corporate finance,A3,1.0,0.0,0.3333333333333333,minutes | |
| 590 | +corporate finance,A3,1.5,0.0,0.3333333333333333,minutes | |
| 591 | +corporate finance,A3,2.0,0.0,0.3333333333333333,minutes | |
| 592 | +corporate finance,A3,3.0,0.0,0.3333333333333333,minutes | |
| 593 | +corporate finance,A3,4.0,0.0,0.3333333333333333,minutes | |
| 594 | +corporate finance,A4,0.25,0.0,0.0,minutes | |
| 595 | +corporate finance,A4,0.5,0.0,0.0,minutes | |
| 596 | +corporate finance,A4,0.75,0.0,0.6666666666666666,minutes | |
| 597 | +corporate finance,A4,1.0,0.0,1.0,minutes | |
| 598 | +corporate finance,A4,1.5,0.0,1.0,minutes | |
| 599 | +corporate finance,A4,2.0,0.0,1.0,minutes | |
| 600 | +corporate finance,A4,3.0,0.0,1.0,minutes | |
| 601 | +corporate finance,A4,4.0,0.0,1.0,minutes | |
| 602 | +executive administration,A0,0.25,0.0,0.0,minutes | |
| 603 | +executive administration,A0,0.5,0.0,0.0,minutes | |
| 604 | +executive administration,A0,0.75,0.0,0.0,minutes | |
| 605 | +executive administration,A0,1.0,0.0,0.0,minutes | |
| 606 | +executive administration,A0,1.5,0.0,0.0,minutes | |
| 607 | +executive administration,A0,2.0,0.0,0.0,minutes | |
| 608 | +executive administration,A0,3.0,0.0,0.0,minutes | |
| 609 | +executive administration,A0,4.0,0.0,0.0,minutes | |
| 610 | +executive administration,A1,0.25,0.0,0.0,minutes | |
| 611 | +executive administration,A1,0.5,0.0,0.0,minutes | |
| 612 | +executive administration,A1,0.75,0.0,0.0,minutes | |
| 613 | +executive administration,A1,1.0,0.0,0.0,minutes | |
| 614 | +executive administration,A1,1.5,0.0,0.0,minutes | |
| 615 | +executive administration,A1,2.0,0.0,0.0,minutes | |
| 616 | +executive administration,A1,3.0,0.0,0.0,minutes | |
| 617 | +executive administration,A1,4.0,0.0,0.0,minutes | |
| 618 | +executive administration,A2,0.25,0.0,0.0,minutes | |
| 619 | +executive administration,A2,0.5,0.0,0.0,minutes | |
| 620 | +executive administration,A2,0.75,0.0,0.0,minutes | |
| 621 | +executive administration,A2,1.0,0.0,0.0,minutes | |
| 622 | +executive administration,A2,1.5,0.0,0.0,minutes | |
| 623 | +executive administration,A2,2.0,0.0,0.0,minutes | |
| 624 | +executive administration,A2,3.0,0.0,0.0,minutes | |
| 625 | +executive administration,A2,4.0,0.0,0.0,minutes | |
| 626 | +executive administration,A3,0.25,0.0,0.0,minutes | |
| 627 | +executive administration,A3,0.5,0.0,0.0,minutes | |
| 628 | +executive administration,A3,0.75,0.0,0.0,minutes | |
| 629 | +executive administration,A3,1.0,0.0,0.0,minutes | |
| 630 | +executive administration,A3,1.5,0.0,0.0,minutes | |
| 631 | +executive administration,A3,2.0,0.0,0.0,minutes | |
| 632 | +executive administration,A3,3.0,0.0,0.0,minutes | |
| 633 | +executive administration,A3,4.0,0.0,0.0,minutes | |
| 634 | +executive administration,A4,0.25,0.0,0.0,minutes | |
| 635 | +executive administration,A4,0.5,0.0,0.0,minutes | |
| 636 | +executive administration,A4,0.75,0.0,0.0,minutes | |
| 637 | +executive administration,A4,1.0,0.0,0.0,minutes | |
| 638 | +executive administration,A4,1.5,0.0,0.0,minutes | |
| 639 | +executive administration,A4,2.0,0.0,0.0,minutes | |
| 640 | +executive administration,A4,3.0,0.0,0.0,minutes | |
| 641 | +executive administration,A4,4.0,0.0,0.0,minutes | |
| 642 | +human resources,A0,0.25,0.0,0.0,minutes | |
| 643 | +human resources,A0,0.5,0.0,0.0,minutes | |
| 644 | +human resources,A0,0.75,0.0,0.0,minutes | |
| 645 | +human resources,A0,1.0,0.0,0.0,minutes | |
| 646 | +human resources,A0,1.5,0.0,0.0,minutes | |
| 647 | +human resources,A0,2.0,0.0,0.0,minutes | |
| 648 | +human resources,A0,3.0,0.0,0.0,minutes | |
| 649 | +human resources,A0,4.0,0.0,0.0,minutes | |
| 650 | +human resources,A1,0.25,0.0,0.0,minutes | |
| 651 | +human resources,A1,0.5,0.0,0.0,minutes | |
| 652 | +human resources,A1,0.75,0.0,0.0,minutes | |
| 653 | +human resources,A1,1.0,0.0,0.0,minutes | |
| 654 | +human resources,A1,1.5,0.0,0.0,minutes | |
| 655 | +human resources,A1,2.0,0.0,0.0,minutes | |
| 656 | +human resources,A1,3.0,0.0,0.0,minutes | |
| 657 | +human resources,A1,4.0,0.0,0.0,minutes | |
| 658 | +human resources,A2,0.25,0.0,0.0,minutes | |
| 659 | +human resources,A2,0.5,0.0,0.0,minutes | |
| 660 | +human resources,A2,0.75,0.0,0.0,minutes | |
| 661 | +human resources,A2,1.0,0.0,0.0,minutes | |
| 662 | +human resources,A2,1.5,0.0,0.0,minutes | |
| 663 | +human resources,A2,2.0,0.0,0.0,minutes | |
| 664 | +human resources,A2,3.0,0.0,0.0,minutes | |
| 665 | +human resources,A2,4.0,0.0,0.0,minutes | |
| 666 | +human resources,A3,0.25,0.0,0.0,minutes | |
| 667 | +human resources,A3,0.5,0.0,0.0,minutes | |
| 668 | +human resources,A3,0.75,0.0,0.0,minutes | |
| 669 | +human resources,A3,1.0,0.0,0.0,minutes | |
| 670 | +human resources,A3,1.5,0.0,0.0,minutes | |
| 671 | +human resources,A3,2.0,0.0,0.0,minutes | |
| 672 | +human resources,A3,3.0,0.0,0.0,minutes | |
| 673 | +human resources,A3,4.0,0.0,0.0,minutes | |
| 674 | +human resources,A4,0.25,0.0,0.0,minutes | |
| 675 | +human resources,A4,0.5,0.0,0.0,minutes | |
| 676 | +human resources,A4,0.75,0.0,0.0,minutes | |
| 677 | +human resources,A4,1.0,0.0,0.0,minutes | |
| 678 | +human resources,A4,1.5,0.0,0.0,minutes | |
| 679 | +human resources,A4,2.0,0.0,0.0,minutes | |
| 680 | +human resources,A4,3.0,0.0,0.0,minutes | |
| 681 | +human resources,A4,4.0,0.0,0.0,minutes | |
| 682 | +legal services,A0,0.25,0.0,0.0,minutes | |
| 683 | +legal services,A0,0.5,0.0,0.0,minutes | |
| 684 | +legal services,A0,0.75,0.0,0.0,minutes | |
| 685 | +legal services,A0,1.0,0.0,0.0,minutes | |
| 686 | +legal services,A0,1.5,0.0,0.0,minutes | |
| 687 | +legal services,A0,2.0,0.0,0.0,minutes | |
| 688 | +legal services,A0,3.0,0.0,0.0,minutes | |
| 689 | +legal services,A0,4.0,0.0,0.0,minutes | |
| 690 | +legal services,A1,0.25,0.0,0.0,minutes | |
| 691 | +legal services,A1,0.5,0.0,0.0,minutes | |
| 692 | +legal services,A1,0.75,0.0,0.0,minutes | |
| 693 | +legal services,A1,1.0,0.0,0.0,minutes | |
| 694 | +legal services,A1,1.5,0.0,0.0,minutes | |
| 695 | +legal services,A1,2.0,0.0,0.0,minutes | |
| 696 | +legal services,A1,3.0,0.0,0.0,minutes | |
| 697 | +legal services,A1,4.0,0.0,0.0,minutes | |
| 698 | +legal services,A2,0.25,0.0,0.0,minutes | |
| 699 | +legal services,A2,0.5,0.0,0.0,minutes | |
| 700 | +legal services,A2,0.75,0.0,0.0,minutes | |
| 701 | +legal services,A2,1.0,0.0,0.0,minutes | |
| 702 | +legal services,A2,1.5,0.0,0.0,minutes | |
| 703 | +legal services,A2,2.0,0.0,0.0,minutes | |
| 704 | +legal services,A2,3.0,0.0,0.0,minutes | |
| 705 | +legal services,A2,4.0,0.0,0.0,minutes | |
| 706 | +legal services,A3,0.25,0.0,0.0,minutes | |
| 707 | +legal services,A3,0.5,0.0,0.0,minutes | |
| 708 | +legal services,A3,0.75,0.0,0.0,minutes | |
| 709 | +legal services,A3,1.0,0.0,0.0,minutes | |
| 710 | +legal services,A3,1.5,0.0,0.0,minutes | |
| 711 | +legal services,A3,2.0,0.0,0.0,minutes | |
| 712 | +legal services,A3,3.0,0.0,0.0,minutes | |
| 713 | +legal services,A3,4.0,0.0,0.0,minutes | |
| 714 | +legal services,A4,0.25,0.0,0.0,minutes | |
| 715 | +legal services,A4,0.5,0.0,0.0,minutes | |
| 716 | +legal services,A4,0.75,0.0,0.0,minutes | |
| 717 | +legal services,A4,1.0,0.0,0.0,minutes | |
| 718 | +legal services,A4,1.5,0.0,0.0,minutes | |
| 719 | +legal services,A4,2.0,0.0,0.0,minutes | |
| 720 | +legal services,A4,3.0,0.0,0.0,minutes | |
| 721 | +legal services,A4,4.0,0.0,0.0,minutes | |
| 722 | +procurement,A0,0.25,0.0,0.0,minutes | |
| 723 | +procurement,A0,0.5,0.0,0.0,minutes | |
| 724 | +procurement,A0,0.75,0.0,0.0,minutes | |
| 725 | +procurement,A0,1.0,0.0,0.0,minutes | |
| 726 | +procurement,A0,1.5,0.0,0.0,minutes | |
| 727 | +procurement,A0,2.0,0.0,0.0,minutes | |
| 728 | +procurement,A0,3.0,0.0,0.0,minutes | |
| 729 | +procurement,A0,4.0,0.0,0.0,minutes | |
| 730 | +procurement,A1,0.25,0.0,0.0,minutes | |
| 731 | +procurement,A1,0.5,0.0,0.0,minutes | |
| 732 | +procurement,A1,0.75,0.0,0.0,minutes | |
| 733 | +procurement,A1,1.0,0.0,0.0,minutes | |
| 734 | +procurement,A1,1.5,0.0,0.0,minutes | |
| 735 | +procurement,A1,2.0,0.0,0.0,minutes | |
| 736 | +procurement,A1,3.0,0.0,0.0,minutes | |
| 737 | +procurement,A1,4.0,0.0,0.0,minutes | |
| 738 | +procurement,A2,0.25,0.0,0.0,minutes | |
| 739 | +procurement,A2,0.5,0.0,0.0,minutes | |
| 740 | +procurement,A2,0.75,0.0,0.0,minutes | |
| 741 | +procurement,A2,1.0,0.0,0.0,minutes | |
| 742 | +procurement,A2,1.5,0.0,0.0,minutes | |
| 743 | +procurement,A2,2.0,0.0,0.0,minutes | |
| 744 | +procurement,A2,3.0,0.0,0.0,minutes | |
| 745 | +procurement,A2,4.0,0.0,0.0,minutes | |
| 746 | +procurement,A3,0.25,0.0,0.0,minutes | |
| 747 | +procurement,A3,0.5,0.0,0.0,minutes | |
| 748 | +procurement,A3,0.75,0.0,0.0,minutes | |
| 749 | +procurement,A3,1.0,0.0,0.0,minutes | |
| 750 | +procurement,A3,1.5,0.0,0.0,minutes | |
| 751 | +procurement,A3,2.0,0.0,0.0,minutes | |
| 752 | +procurement,A3,3.0,0.0,0.0,minutes | |
| 753 | +procurement,A3,4.0,0.0,0.0,minutes | |
| 754 | +procurement,A4,0.25,0.0,0.0,minutes | |
| 755 | +procurement,A4,0.5,0.0,0.0,minutes | |
| 756 | +procurement,A4,0.75,0.0,0.0,minutes | |
| 757 | +procurement,A4,1.0,0.0,0.0,minutes | |
| 758 | +procurement,A4,1.5,0.0,0.0,minutes | |
| 759 | +procurement,A4,2.0,0.0,0.0,minutes | |
| 760 | +procurement,A4,3.0,0.0,0.0,minutes | |
| 761 | +procurement,A4,4.0,0.0,0.0,minutes | |
| 762 | +real estate appraisal,A0,0.25,0.0,0.0,minutes | |
| 763 | +real estate appraisal,A0,0.5,0.0,0.0,minutes | |
| 764 | +real estate appraisal,A0,0.75,0.0,0.0,minutes | |
| 765 | +real estate appraisal,A0,1.0,0.0,0.0,minutes | |
| 766 | +real estate appraisal,A0,1.5,0.0,0.0,minutes | |
| 767 | +real estate appraisal,A0,2.0,0.0,0.0,minutes | |
| 768 | +real estate appraisal,A0,3.0,0.0,0.0,minutes | |
| 769 | +real estate appraisal,A0,4.0,0.0,0.0,minutes | |
| 770 | +real estate appraisal,A1,0.25,0.0,0.0,minutes | |
| 771 | +real estate appraisal,A1,0.5,0.0,0.0,minutes | |
| 772 | +real estate appraisal,A1,0.75,0.0,0.0,minutes | |
| 773 | +real estate appraisal,A1,1.0,0.0,0.0,minutes | |
| 774 | +real estate appraisal,A1,1.5,0.0,0.0,minutes | |
| 775 | +real estate appraisal,A1,2.0,0.0,0.0,minutes | |
| 776 | +real estate appraisal,A1,3.0,0.0,0.0,minutes | |
| 777 | +real estate appraisal,A1,4.0,0.0,0.0,minutes | |
| 778 | +real estate appraisal,A2,0.25,0.0,0.0,minutes | |
| 779 | +real estate appraisal,A2,0.5,0.0,0.0,minutes | |
| 780 | +real estate appraisal,A2,0.75,0.0,0.3333333333333333,minutes | |
| 781 | +real estate appraisal,A2,1.0,0.0,0.3333333333333333,minutes | |
| 782 | +real estate appraisal,A2,1.5,0.0,0.3333333333333333,minutes | |
| 783 | +real estate appraisal,A2,2.0,0.0,0.3333333333333333,minutes | |
| 784 | +real estate appraisal,A2,3.0,0.0,0.3333333333333333,minutes | |
| 785 | +real estate appraisal,A2,4.0,0.0,0.3333333333333333,minutes | |
| 786 | +real estate appraisal,A3,0.25,0.0,0.0,minutes | |
| 787 | +real estate appraisal,A3,0.5,0.0,0.0,minutes | |
| 788 | +real estate appraisal,A3,0.75,0.0,0.0,minutes | |
| 789 | +real estate appraisal,A3,1.0,0.0,0.0,minutes | |
| 790 | +real estate appraisal,A3,1.5,0.0,0.0,minutes | |
| 791 | +real estate appraisal,A3,2.0,0.0,0.0,minutes | |
| 792 | +real estate appraisal,A3,3.0,0.0,0.0,minutes | |
| 793 | +real estate appraisal,A3,4.0,0.0,0.0,minutes | |
| 794 | +real estate appraisal,A4,0.25,0.0,0.0,minutes | |
| 795 | +real estate appraisal,A4,0.5,0.0,0.0,minutes | |
| 796 | +real estate appraisal,A4,0.75,0.0,0.0,minutes | |
| 797 | +real estate appraisal,A4,1.0,0.0,0.0,minutes | |
| 798 | +real estate appraisal,A4,1.5,0.0,0.0,minutes | |
| 799 | +real estate appraisal,A4,2.0,0.0,0.0,minutes | |
| 800 | +real estate appraisal,A4,3.0,0.0,0.0,minutes | |
| 801 | +real estate appraisal,A4,4.0,0.0,0.0,minutes | |
| 802 | +academic research,A0,0.25,15.0,0.0,equal_H15 | |
| 803 | +academic research,A0,0.5,15.0,0.0,equal_H15 | |
| 804 | +academic research,A0,0.75,15.0,0.0,equal_H15 | |
| 805 | +academic research,A0,1.0,15.0,0.0,equal_H15 | |
| 806 | +academic research,A0,1.5,15.0,0.0,equal_H15 | |
| 807 | +academic research,A0,2.0,15.0,0.0,equal_H15 | |
| 808 | +academic research,A0,3.0,15.0,0.0,equal_H15 | |
| 809 | +academic research,A0,4.0,15.0,0.0,equal_H15 | |
| 810 | +academic research,A1,0.25,15.0,0.0,equal_H15 | |
| 811 | +academic research,A1,0.5,15.0,0.0,equal_H15 | |
| 812 | +academic research,A1,0.75,15.0,1.0,equal_H15 | |
| 813 | +academic research,A1,1.0,15.0,1.0,equal_H15 | |
| 814 | +academic research,A1,1.5,15.0,1.0,equal_H15 | |
| 815 | +academic research,A1,2.0,15.0,1.0,equal_H15 | |
| 816 | +academic research,A1,3.0,15.0,1.0,equal_H15 | |
| 817 | +academic research,A1,4.0,15.0,1.0,equal_H15 | |
| 818 | +academic research,A2,0.25,15.0,0.0,equal_H15 | |
| 819 | +academic research,A2,0.5,15.0,0.0,equal_H15 | |
| 820 | +academic research,A2,0.75,15.0,1.0,equal_H15 | |
| 821 | +academic research,A2,1.0,15.0,1.0,equal_H15 | |
| 822 | +academic research,A2,1.5,15.0,1.0,equal_H15 | |
| 823 | +academic research,A2,2.0,15.0,1.0,equal_H15 | |
| 824 | +academic research,A2,3.0,15.0,1.0,equal_H15 | |
| 825 | +academic research,A2,4.0,15.0,1.0,equal_H15 | |
| 826 | +academic research,A3,0.25,15.0,0.0,equal_H15 | |
| 827 | +academic research,A3,0.5,15.0,0.0,equal_H15 | |
| 828 | +academic research,A3,0.75,15.0,0.6666666666666666,equal_H15 | |
| 829 | +academic research,A3,1.0,15.0,1.0,equal_H15 | |
| 830 | +academic research,A3,1.5,15.0,1.0,equal_H15 | |
| 831 | +academic research,A3,2.0,15.0,1.0,equal_H15 | |
| 832 | +academic research,A3,3.0,15.0,1.0,equal_H15 | |
| 833 | +academic research,A3,4.0,15.0,1.0,equal_H15 | |
| 834 | +academic research,A4,0.25,15.0,0.0,equal_H15 | |
| 835 | +academic research,A4,0.5,15.0,0.0,equal_H15 | |
| 836 | +academic research,A4,0.75,15.0,0.6666666666666666,equal_H15 | |
| 837 | +academic research,A4,1.0,15.0,1.0,equal_H15 | |
| 838 | +academic research,A4,1.5,15.0,1.0,equal_H15 | |
| 839 | +academic research,A4,2.0,15.0,1.0,equal_H15 | |
| 840 | +academic research,A4,3.0,15.0,1.0,equal_H15 | |
| 841 | +academic research,A4,4.0,15.0,1.0,equal_H15 | |
| 842 | +accounting,A0,0.25,15.0,0.0,equal_H15 | |
| 843 | +accounting,A0,0.5,15.0,0.0,equal_H15 | |
| 844 | +accounting,A0,0.75,15.0,0.0,equal_H15 | |
| 845 | +accounting,A0,1.0,15.0,0.0,equal_H15 | |
| 846 | +accounting,A0,1.5,15.0,0.0,equal_H15 | |
| 847 | +accounting,A0,2.0,15.0,0.0,equal_H15 | |
| 848 | +accounting,A0,3.0,15.0,0.0,equal_H15 | |
| 849 | +accounting,A0,4.0,15.0,0.0,equal_H15 | |
| 850 | +accounting,A1,0.25,15.0,0.0,equal_H15 | |
| 851 | +accounting,A1,0.5,15.0,0.0,equal_H15 | |
| 852 | +accounting,A1,0.75,15.0,0.0,equal_H15 | |
| 853 | +accounting,A1,1.0,15.0,0.3333333333333333,equal_H15 | |
| 854 | +accounting,A1,1.5,15.0,1.0,equal_H15 | |
| 855 | +accounting,A1,2.0,15.0,1.0,equal_H15 | |
| 856 | +accounting,A1,3.0,15.0,1.0,equal_H15 | |
| 857 | +accounting,A1,4.0,15.0,1.0,equal_H15 | |
| 858 | +accounting,A2,0.25,15.0,0.0,equal_H15 | |
| 859 | +accounting,A2,0.5,15.0,0.0,equal_H15 | |
| 860 | +accounting,A2,0.75,15.0,0.0,equal_H15 | |
| 861 | +accounting,A2,1.0,15.0,0.6666666666666666,equal_H15 | |
| 862 | +accounting,A2,1.5,15.0,1.0,equal_H15 | |
| 863 | +accounting,A2,2.0,15.0,1.0,equal_H15 | |
| 864 | +accounting,A2,3.0,15.0,1.0,equal_H15 | |
| 865 | +accounting,A2,4.0,15.0,1.0,equal_H15 | |
| 866 | +accounting,A3,0.25,15.0,0.0,equal_H15 | |
| 867 | +accounting,A3,0.5,15.0,0.0,equal_H15 | |
| 868 | +accounting,A3,0.75,15.0,0.0,equal_H15 | |
| 869 | +accounting,A3,1.0,15.0,0.0,equal_H15 | |
| 870 | +accounting,A3,1.5,15.0,1.0,equal_H15 | |
| 871 | +accounting,A3,2.0,15.0,1.0,equal_H15 | |
| 872 | +accounting,A3,3.0,15.0,1.0,equal_H15 | |
| 873 | +accounting,A3,4.0,15.0,1.0,equal_H15 | |
| 874 | +accounting,A4,0.25,15.0,0.0,equal_H15 | |
| 875 | +accounting,A4,0.5,15.0,0.0,equal_H15 | |
| 876 | +accounting,A4,0.75,15.0,0.0,equal_H15 | |
| 877 | +accounting,A4,1.0,15.0,0.0,equal_H15 | |
| 878 | +accounting,A4,1.5,15.0,1.0,equal_H15 | |
| 879 | +accounting,A4,2.0,15.0,1.0,equal_H15 | |
| 880 | +accounting,A4,3.0,15.0,1.0,equal_H15 | |
| 881 | +accounting,A4,4.0,15.0,1.0,equal_H15 | |
| 882 | +banking,A0,0.25,15.0,0.0,equal_H15 | |
| 883 | +banking,A0,0.5,15.0,0.0,equal_H15 | |
| 884 | +banking,A0,0.75,15.0,0.0,equal_H15 | |
| 885 | +banking,A0,1.0,15.0,0.0,equal_H15 | |
| 886 | +banking,A0,1.5,15.0,0.0,equal_H15 | |
| 887 | +banking,A0,2.0,15.0,0.0,equal_H15 | |
| 888 | +banking,A0,3.0,15.0,0.0,equal_H15 | |
| 889 | +banking,A0,4.0,15.0,0.0,equal_H15 | |
| 890 | +banking,A1,0.25,15.0,0.0,equal_H15 | |
| 891 | +banking,A1,0.5,15.0,0.0,equal_H15 | |
| 892 | +banking,A1,0.75,15.0,0.0,equal_H15 | |
| 893 | +banking,A1,1.0,15.0,0.3333333333333333,equal_H15 | |
| 894 | +banking,A1,1.5,15.0,0.3333333333333333,equal_H15 | |
| 895 | +banking,A1,2.0,15.0,0.3333333333333333,equal_H15 | |
| 896 | +banking,A1,3.0,15.0,0.3333333333333333,equal_H15 | |
| 897 | +banking,A1,4.0,15.0,0.3333333333333333,equal_H15 | |
| 898 | +banking,A2,0.25,15.0,0.0,equal_H15 | |
| 899 | +banking,A2,0.5,15.0,0.0,equal_H15 | |
| 900 | +banking,A2,0.75,15.0,0.0,equal_H15 | |
| 901 | +banking,A2,1.0,15.0,0.3333333333333333,equal_H15 | |
| 902 | +banking,A2,1.5,15.0,0.3333333333333333,equal_H15 | |
| 903 | +banking,A2,2.0,15.0,0.3333333333333333,equal_H15 | |
| 904 | +banking,A2,3.0,15.0,0.3333333333333333,equal_H15 | |
| 905 | +banking,A2,4.0,15.0,0.3333333333333333,equal_H15 | |
| 906 | +banking,A3,0.25,15.0,0.0,equal_H15 | |
| 907 | +banking,A3,0.5,15.0,0.0,equal_H15 | |
| 908 | +banking,A3,0.75,15.0,0.0,equal_H15 | |
| 909 | +banking,A3,1.0,15.0,0.6666666666666666,equal_H15 | |
| 910 | +banking,A3,1.5,15.0,0.6666666666666666,equal_H15 | |
| 911 | +banking,A3,2.0,15.0,0.6666666666666666,equal_H15 | |
| 912 | +banking,A3,3.0,15.0,0.6666666666666666,equal_H15 | |
| 913 | +banking,A3,4.0,15.0,0.6666666666666666,equal_H15 | |
| 914 | +banking,A4,0.25,15.0,0.0,equal_H15 | |
| 915 | +banking,A4,0.5,15.0,0.0,equal_H15 | |
| 916 | +banking,A4,0.75,15.0,0.0,equal_H15 | |
| 917 | +banking,A4,1.0,15.0,1.0,equal_H15 | |
| 918 | +banking,A4,1.5,15.0,1.0,equal_H15 | |
| 919 | +banking,A4,2.0,15.0,1.0,equal_H15 | |
| 920 | +banking,A4,3.0,15.0,1.0,equal_H15 | |
| 921 | +banking,A4,4.0,15.0,1.0,equal_H15 | |
| 922 | +business intelligence,A0,0.25,15.0,0.0,equal_H15 | |
| 923 | +business intelligence,A0,0.5,15.0,0.0,equal_H15 | |
| 924 | +business intelligence,A0,0.75,15.0,0.0,equal_H15 | |
| 925 | +business intelligence,A0,1.0,15.0,0.0,equal_H15 | |
| 926 | +business intelligence,A0,1.5,15.0,0.0,equal_H15 | |
| 927 | +business intelligence,A0,2.0,15.0,0.0,equal_H15 | |
| 928 | +business intelligence,A0,3.0,15.0,0.0,equal_H15 | |
| 929 | +business intelligence,A0,4.0,15.0,0.0,equal_H15 | |
| 930 | +business intelligence,A1,0.25,15.0,0.0,equal_H15 | |
| 931 | +business intelligence,A1,0.5,15.0,0.0,equal_H15 | |
| 932 | +business intelligence,A1,0.75,15.0,0.0,equal_H15 | |
| 933 | +business intelligence,A1,1.0,15.0,0.6666666666666666,equal_H15 | |
| 934 | +business intelligence,A1,1.5,15.0,0.6666666666666666,equal_H15 | |
| 935 | +business intelligence,A1,2.0,15.0,0.6666666666666666,equal_H15 | |
| 936 | +business intelligence,A1,3.0,15.0,0.6666666666666666,equal_H15 | |
| 937 | +business intelligence,A1,4.0,15.0,0.6666666666666666,equal_H15 | |
| 938 | +business intelligence,A2,0.25,15.0,0.0,equal_H15 | |
| 939 | +business intelligence,A2,0.5,15.0,0.0,equal_H15 | |
| 940 | +business intelligence,A2,0.75,15.0,0.0,equal_H15 | |
| 941 | +business intelligence,A2,1.0,15.0,0.6666666666666666,equal_H15 | |
| 942 | +business intelligence,A2,1.5,15.0,1.0,equal_H15 | |
| 943 | +business intelligence,A2,2.0,15.0,1.0,equal_H15 | |
| 944 | +business intelligence,A2,3.0,15.0,1.0,equal_H15 | |
| 945 | +business intelligence,A2,4.0,15.0,1.0,equal_H15 | |
| 946 | +business intelligence,A3,0.25,15.0,0.0,equal_H15 | |
| 947 | +business intelligence,A3,0.5,15.0,0.0,equal_H15 | |
| 948 | +business intelligence,A3,0.75,15.0,0.0,equal_H15 | |
| 949 | +business intelligence,A3,1.0,15.0,0.3333333333333333,equal_H15 | |
| 950 | +business intelligence,A3,1.5,15.0,1.0,equal_H15 | |
| 951 | +business intelligence,A3,2.0,15.0,1.0,equal_H15 | |
| 952 | +business intelligence,A3,3.0,15.0,1.0,equal_H15 | |
| 953 | +business intelligence,A3,4.0,15.0,1.0,equal_H15 | |
| 954 | +business intelligence,A4,0.25,15.0,0.0,equal_H15 | |
| 955 | +business intelligence,A4,0.5,15.0,0.0,equal_H15 | |
| 956 | +business intelligence,A4,0.75,15.0,0.0,equal_H15 | |
| 957 | +business intelligence,A4,1.0,15.0,0.0,equal_H15 | |
| 958 | +business intelligence,A4,1.5,15.0,0.6666666666666666,equal_H15 | |
| 959 | +business intelligence,A4,2.0,15.0,0.6666666666666666,equal_H15 | |
| 960 | +business intelligence,A4,3.0,15.0,0.6666666666666666,equal_H15 | |
| 961 | +business intelligence,A4,4.0,15.0,0.6666666666666666,equal_H15 | |
| 962 | +corporate finance,A0,0.25,15.0,0.0,equal_H15 | |
| 963 | +corporate finance,A0,0.5,15.0,0.0,equal_H15 | |
| 964 | +corporate finance,A0,0.75,15.0,0.0,equal_H15 | |
| 965 | +corporate finance,A0,1.0,15.0,0.0,equal_H15 | |
| 966 | +corporate finance,A0,1.5,15.0,0.0,equal_H15 | |
| 967 | +corporate finance,A0,2.0,15.0,0.0,equal_H15 | |
| 968 | +corporate finance,A0,3.0,15.0,0.0,equal_H15 | |
| 969 | +corporate finance,A0,4.0,15.0,0.0,equal_H15 | |
| 970 | +corporate finance,A1,0.25,15.0,0.0,equal_H15 | |
| 971 | +corporate finance,A1,0.5,15.0,0.0,equal_H15 | |
| 972 | +corporate finance,A1,0.75,15.0,0.3333333333333333,equal_H15 | |
| 973 | +corporate finance,A1,1.0,15.0,1.0,equal_H15 | |
| 974 | +corporate finance,A1,1.5,15.0,1.0,equal_H15 | |
| 975 | +corporate finance,A1,2.0,15.0,1.0,equal_H15 | |
| 976 | +corporate finance,A1,3.0,15.0,1.0,equal_H15 | |
| 977 | +corporate finance,A1,4.0,15.0,1.0,equal_H15 | |
| 978 | +corporate finance,A2,0.25,15.0,0.0,equal_H15 | |
| 979 | +corporate finance,A2,0.5,15.0,0.0,equal_H15 | |
| 980 | +corporate finance,A2,0.75,15.0,0.6666666666666666,equal_H15 | |
| 981 | +corporate finance,A2,1.0,15.0,1.0,equal_H15 | |
| 982 | +corporate finance,A2,1.5,15.0,1.0,equal_H15 | |
| 983 | +corporate finance,A2,2.0,15.0,1.0,equal_H15 | |
| 984 | +corporate finance,A2,3.0,15.0,1.0,equal_H15 | |
| 985 | +corporate finance,A2,4.0,15.0,1.0,equal_H15 | |
| 986 | +corporate finance,A3,0.25,15.0,0.0,equal_H15 | |
| 987 | +corporate finance,A3,0.5,15.0,0.0,equal_H15 | |
| 988 | +corporate finance,A3,0.75,15.0,1.0,equal_H15 | |
| 989 | +corporate finance,A3,1.0,15.0,1.0,equal_H15 | |
| 990 | +corporate finance,A3,1.5,15.0,1.0,equal_H15 | |
| 991 | +corporate finance,A3,2.0,15.0,1.0,equal_H15 | |
| 992 | +corporate finance,A3,3.0,15.0,1.0,equal_H15 | |
| 993 | +corporate finance,A3,4.0,15.0,1.0,equal_H15 | |
| 994 | +corporate finance,A4,0.25,15.0,0.0,equal_H15 | |
| 995 | +corporate finance,A4,0.5,15.0,0.0,equal_H15 | |
| 996 | +corporate finance,A4,0.75,15.0,0.6666666666666666,equal_H15 | |
| 997 | +corporate finance,A4,1.0,15.0,1.0,equal_H15 | |
| 998 | +corporate finance,A4,1.5,15.0,1.0,equal_H15 | |
| 999 | +corporate finance,A4,2.0,15.0,1.0,equal_H15 | |
| 1000 | +corporate finance,A4,3.0,15.0,1.0,equal_H15 | |
| 1001 | +corporate finance,A4,4.0,15.0,1.0,equal_H15 | |
| 1002 | +executive administration,A0,0.25,15.0,0.0,equal_H15 | |
| 1003 | +executive administration,A0,0.5,15.0,0.0,equal_H15 | |
| 1004 | +executive administration,A0,0.75,15.0,0.0,equal_H15 | |
| 1005 | +executive administration,A0,1.0,15.0,0.0,equal_H15 | |
| 1006 | +executive administration,A0,1.5,15.0,0.0,equal_H15 | |
| 1007 | +executive administration,A0,2.0,15.0,0.0,equal_H15 | |
| 1008 | +executive administration,A0,3.0,15.0,0.0,equal_H15 | |
| 1009 | +executive administration,A0,4.0,15.0,0.0,equal_H15 | |
| 1010 | +executive administration,A1,0.25,15.0,0.0,equal_H15 | |
| 1011 | +executive administration,A1,0.5,15.0,0.0,equal_H15 | |
| 1012 | +executive administration,A1,0.75,15.0,0.0,equal_H15 | |
| 1013 | +executive administration,A1,1.0,15.0,0.0,equal_H15 | |
| 1014 | +executive administration,A1,1.5,15.0,0.0,equal_H15 | |
| 1015 | +executive administration,A1,2.0,15.0,0.0,equal_H15 | |
| 1016 | +executive administration,A1,3.0,15.0,0.0,equal_H15 | |
| 1017 | +executive administration,A1,4.0,15.0,0.0,equal_H15 | |
| 1018 | +executive administration,A2,0.25,15.0,0.0,equal_H15 | |
| 1019 | +executive administration,A2,0.5,15.0,0.0,equal_H15 | |
| 1020 | +executive administration,A2,0.75,15.0,0.0,equal_H15 | |
| 1021 | +executive administration,A2,1.0,15.0,0.0,equal_H15 | |
| 1022 | +executive administration,A2,1.5,15.0,0.0,equal_H15 | |
| 1023 | +executive administration,A2,2.0,15.0,0.0,equal_H15 | |
| 1024 | +executive administration,A2,3.0,15.0,0.0,equal_H15 | |
| 1025 | +executive administration,A2,4.0,15.0,0.0,equal_H15 | |
| 1026 | +executive administration,A3,0.25,15.0,0.0,equal_H15 | |
| 1027 | +executive administration,A3,0.5,15.0,0.0,equal_H15 | |
| 1028 | +executive administration,A3,0.75,15.0,0.0,equal_H15 | |
| 1029 | +executive administration,A3,1.0,15.0,0.0,equal_H15 | |
| 1030 | +executive administration,A3,1.5,15.0,0.0,equal_H15 | |
| 1031 | +executive administration,A3,2.0,15.0,0.0,equal_H15 | |
| 1032 | +executive administration,A3,3.0,15.0,0.0,equal_H15 | |
| 1033 | +executive administration,A3,4.0,15.0,0.0,equal_H15 | |
| 1034 | +executive administration,A4,0.25,15.0,0.0,equal_H15 | |
| 1035 | +executive administration,A4,0.5,15.0,0.0,equal_H15 | |
| 1036 | +executive administration,A4,0.75,15.0,0.0,equal_H15 | |
| 1037 | +executive administration,A4,1.0,15.0,0.0,equal_H15 | |
| 1038 | +executive administration,A4,1.5,15.0,0.0,equal_H15 | |
| 1039 | +executive administration,A4,2.0,15.0,0.0,equal_H15 | |
| 1040 | +executive administration,A4,3.0,15.0,0.0,equal_H15 | |
| 1041 | +executive administration,A4,4.0,15.0,0.0,equal_H15 | |
| 1042 | +human resources,A0,0.25,15.0,0.0,equal_H15 | |
| 1043 | +human resources,A0,0.5,15.0,0.0,equal_H15 | |
| 1044 | +human resources,A0,0.75,15.0,0.0,equal_H15 | |
| 1045 | +human resources,A0,1.0,15.0,0.0,equal_H15 | |
| 1046 | +human resources,A0,1.5,15.0,0.0,equal_H15 | |
| 1047 | +human resources,A0,2.0,15.0,0.0,equal_H15 | |
| 1048 | +human resources,A0,3.0,15.0,0.0,equal_H15 | |
| 1049 | +human resources,A0,4.0,15.0,0.0,equal_H15 | |
| 1050 | +human resources,A1,0.25,15.0,0.0,equal_H15 | |
| 1051 | +human resources,A1,0.5,15.0,0.0,equal_H15 | |
| 1052 | +human resources,A1,0.75,15.0,0.0,equal_H15 | |
| 1053 | +human resources,A1,1.0,15.0,0.0,equal_H15 | |
| 1054 | +human resources,A1,1.5,15.0,0.0,equal_H15 | |
| 1055 | +human resources,A1,2.0,15.0,0.0,equal_H15 | |
| 1056 | +human resources,A1,3.0,15.0,0.0,equal_H15 | |
| 1057 | +human resources,A1,4.0,15.0,0.0,equal_H15 | |
| 1058 | +human resources,A2,0.25,15.0,0.0,equal_H15 | |
| 1059 | +human resources,A2,0.5,15.0,0.0,equal_H15 | |
| 1060 | +human resources,A2,0.75,15.0,0.0,equal_H15 | |
| 1061 | +human resources,A2,1.0,15.0,0.0,equal_H15 | |
| 1062 | +human resources,A2,1.5,15.0,0.0,equal_H15 | |
| 1063 | +human resources,A2,2.0,15.0,0.0,equal_H15 | |
| 1064 | +human resources,A2,3.0,15.0,0.0,equal_H15 | |
| 1065 | +human resources,A2,4.0,15.0,0.0,equal_H15 | |
| 1066 | +human resources,A3,0.25,15.0,0.0,equal_H15 | |
| 1067 | +human resources,A3,0.5,15.0,0.0,equal_H15 | |
| 1068 | +human resources,A3,0.75,15.0,0.0,equal_H15 | |
| 1069 | +human resources,A3,1.0,15.0,0.6666666666666666,equal_H15 | |
| 1070 | +human resources,A3,1.5,15.0,0.6666666666666666,equal_H15 | |
| 1071 | +human resources,A3,2.0,15.0,0.6666666666666666,equal_H15 | |
| 1072 | +human resources,A3,3.0,15.0,0.6666666666666666,equal_H15 | |
| 1073 | +human resources,A3,4.0,15.0,0.6666666666666666,equal_H15 | |
| 1074 | +human resources,A4,0.25,15.0,0.0,equal_H15 | |
| 1075 | +human resources,A4,0.5,15.0,0.0,equal_H15 | |
| 1076 | +human resources,A4,0.75,15.0,0.0,equal_H15 | |
| 1077 | +human resources,A4,1.0,15.0,0.3333333333333333,equal_H15 | |
| 1078 | +human resources,A4,1.5,15.0,0.6666666666666666,equal_H15 | |
| 1079 | +human resources,A4,2.0,15.0,0.6666666666666666,equal_H15 | |
| 1080 | +human resources,A4,3.0,15.0,0.6666666666666666,equal_H15 | |
| 1081 | +human resources,A4,4.0,15.0,0.6666666666666666,equal_H15 | |
| 1082 | +legal services,A0,0.25,15.0,0.0,equal_H15 | |
| 1083 | +legal services,A0,0.5,15.0,0.0,equal_H15 | |
| 1084 | +legal services,A0,0.75,15.0,0.0,equal_H15 | |
| 1085 | +legal services,A0,1.0,15.0,0.0,equal_H15 | |
| 1086 | +legal services,A0,1.5,15.0,0.0,equal_H15 | |
| 1087 | +legal services,A0,2.0,15.0,0.0,equal_H15 | |
| 1088 | +legal services,A0,3.0,15.0,0.0,equal_H15 | |
| 1089 | +legal services,A0,4.0,15.0,0.0,equal_H15 | |
| 1090 | +legal services,A1,0.25,15.0,0.0,equal_H15 | |
| 1091 | +legal services,A1,0.5,15.0,0.0,equal_H15 | |
| 1092 | +legal services,A1,0.75,15.0,0.0,equal_H15 | |
| 1093 | +legal services,A1,1.0,15.0,0.0,equal_H15 | |
| 1094 | +legal services,A1,1.5,15.0,0.0,equal_H15 | |
| 1095 | +legal services,A1,2.0,15.0,0.0,equal_H15 | |
| 1096 | +legal services,A1,3.0,15.0,0.0,equal_H15 | |
| 1097 | +legal services,A1,4.0,15.0,0.0,equal_H15 | |
| 1098 | +legal services,A2,0.25,15.0,0.0,equal_H15 | |
| 1099 | +legal services,A2,0.5,15.0,0.0,equal_H15 | |
| 1100 | +legal services,A2,0.75,15.0,0.0,equal_H15 | |
| 1101 | +legal services,A2,1.0,15.0,0.0,equal_H15 | |
| 1102 | +legal services,A2,1.5,15.0,0.0,equal_H15 | |
| 1103 | +legal services,A2,2.0,15.0,0.0,equal_H15 | |
| 1104 | +legal services,A2,3.0,15.0,0.0,equal_H15 | |
| 1105 | +legal services,A2,4.0,15.0,0.0,equal_H15 | |
| 1106 | +legal services,A3,0.25,15.0,0.0,equal_H15 | |
| 1107 | +legal services,A3,0.5,15.0,0.0,equal_H15 | |
| 1108 | +legal services,A3,0.75,15.0,0.0,equal_H15 | |
| 1109 | +legal services,A3,1.0,15.0,0.0,equal_H15 | |
| 1110 | +legal services,A3,1.5,15.0,0.0,equal_H15 | |
| 1111 | +legal services,A3,2.0,15.0,0.0,equal_H15 | |
| 1112 | +legal services,A3,3.0,15.0,0.0,equal_H15 | |
| 1113 | +legal services,A3,4.0,15.0,0.0,equal_H15 | |
| 1114 | +legal services,A4,0.25,15.0,0.0,equal_H15 | |
| 1115 | +legal services,A4,0.5,15.0,0.0,equal_H15 | |
| 1116 | +legal services,A4,0.75,15.0,0.0,equal_H15 | |
| 1117 | +legal services,A4,1.0,15.0,0.0,equal_H15 | |
| 1118 | +legal services,A4,1.5,15.0,0.0,equal_H15 | |
| 1119 | +legal services,A4,2.0,15.0,0.0,equal_H15 | |
| 1120 | +legal services,A4,3.0,15.0,0.0,equal_H15 | |
| 1121 | +legal services,A4,4.0,15.0,0.0,equal_H15 | |
| 1122 | +procurement,A0,0.25,15.0,0.0,equal_H15 | |
| 1123 | +procurement,A0,0.5,15.0,0.0,equal_H15 | |
| 1124 | +procurement,A0,0.75,15.0,0.0,equal_H15 | |
| 1125 | +procurement,A0,1.0,15.0,0.0,equal_H15 | |
| 1126 | +procurement,A0,1.5,15.0,0.0,equal_H15 | |
| 1127 | +procurement,A0,2.0,15.0,0.0,equal_H15 | |
| 1128 | +procurement,A0,3.0,15.0,0.0,equal_H15 | |
| 1129 | +procurement,A0,4.0,15.0,0.0,equal_H15 | |
| 1130 | +procurement,A1,0.25,15.0,0.0,equal_H15 | |
| 1131 | +procurement,A1,0.5,15.0,0.0,equal_H15 | |
| 1132 | +procurement,A1,0.75,15.0,0.0,equal_H15 | |
| 1133 | +procurement,A1,1.0,15.0,0.0,equal_H15 | |
| 1134 | +procurement,A1,1.5,15.0,0.0,equal_H15 | |
| 1135 | +procurement,A1,2.0,15.0,0.0,equal_H15 | |
| 1136 | +procurement,A1,3.0,15.0,0.0,equal_H15 | |
| 1137 | +procurement,A1,4.0,15.0,0.0,equal_H15 | |
| 1138 | +procurement,A2,0.25,15.0,0.0,equal_H15 | |
| 1139 | +procurement,A2,0.5,15.0,0.0,equal_H15 | |
| 1140 | +procurement,A2,0.75,15.0,0.0,equal_H15 | |
| 1141 | +procurement,A2,1.0,15.0,0.0,equal_H15 | |
| 1142 | +procurement,A2,1.5,15.0,0.0,equal_H15 | |
| 1143 | +procurement,A2,2.0,15.0,0.0,equal_H15 | |
| 1144 | +procurement,A2,3.0,15.0,0.0,equal_H15 | |
| 1145 | +procurement,A2,4.0,15.0,0.0,equal_H15 | |
| 1146 | +procurement,A3,0.25,15.0,0.0,equal_H15 | |
| 1147 | +procurement,A3,0.5,15.0,0.0,equal_H15 | |
| 1148 | +procurement,A3,0.75,15.0,0.3333333333333333,equal_H15 | |
| 1149 | +procurement,A3,1.0,15.0,0.3333333333333333,equal_H15 | |
| 1150 | +procurement,A3,1.5,15.0,0.3333333333333333,equal_H15 | |
| 1151 | +procurement,A3,2.0,15.0,0.3333333333333333,equal_H15 | |
| 1152 | +procurement,A3,3.0,15.0,0.3333333333333333,equal_H15 | |
| 1153 | +procurement,A3,4.0,15.0,0.3333333333333333,equal_H15 | |
| 1154 | +procurement,A4,0.25,15.0,0.0,equal_H15 | |
| 1155 | +procurement,A4,0.5,15.0,0.0,equal_H15 | |
| 1156 | +procurement,A4,0.75,15.0,0.0,equal_H15 | |
| 1157 | +procurement,A4,1.0,15.0,0.0,equal_H15 | |
| 1158 | +procurement,A4,1.5,15.0,0.0,equal_H15 | |
| 1159 | +procurement,A4,2.0,15.0,0.0,equal_H15 | |
| 1160 | +procurement,A4,3.0,15.0,0.0,equal_H15 | |
| 1161 | +procurement,A4,4.0,15.0,0.0,equal_H15 | |
| 1162 | +real estate appraisal,A0,0.25,15.0,0.0,equal_H15 | |
| 1163 | +real estate appraisal,A0,0.5,15.0,0.0,equal_H15 | |
| 1164 | +real estate appraisal,A0,0.75,15.0,0.0,equal_H15 | |
| 1165 | +real estate appraisal,A0,1.0,15.0,0.0,equal_H15 | |
| 1166 | +real estate appraisal,A0,1.5,15.0,0.0,equal_H15 | |
| 1167 | +real estate appraisal,A0,2.0,15.0,0.0,equal_H15 | |
| 1168 | +real estate appraisal,A0,3.0,15.0,0.0,equal_H15 | |
| 1169 | +real estate appraisal,A0,4.0,15.0,0.0,equal_H15 | |
| 1170 | +real estate appraisal,A1,0.25,15.0,0.0,equal_H15 | |
| 1171 | +real estate appraisal,A1,0.5,15.0,0.0,equal_H15 | |
| 1172 | +real estate appraisal,A1,0.75,15.0,1.0,equal_H15 | |
| 1173 | +real estate appraisal,A1,1.0,15.0,1.0,equal_H15 | |
| 1174 | +real estate appraisal,A1,1.5,15.0,1.0,equal_H15 | |
| 1175 | +real estate appraisal,A1,2.0,15.0,1.0,equal_H15 | |
| 1176 | +real estate appraisal,A1,3.0,15.0,1.0,equal_H15 | |
| 1177 | +real estate appraisal,A1,4.0,15.0,1.0,equal_H15 | |
| 1178 | +real estate appraisal,A2,0.25,15.0,0.0,equal_H15 | |
| 1179 | +real estate appraisal,A2,0.5,15.0,0.0,equal_H15 | |
| 1180 | +real estate appraisal,A2,0.75,15.0,0.6666666666666666,equal_H15 | |
| 1181 | +real estate appraisal,A2,1.0,15.0,0.6666666666666666,equal_H15 | |
| 1182 | +real estate appraisal,A2,1.5,15.0,0.6666666666666666,equal_H15 | |
| 1183 | +real estate appraisal,A2,2.0,15.0,0.6666666666666666,equal_H15 | |
| 1184 | +real estate appraisal,A2,3.0,15.0,0.6666666666666666,equal_H15 | |
| 1185 | +real estate appraisal,A2,4.0,15.0,0.6666666666666666,equal_H15 | |
| 1186 | +real estate appraisal,A3,0.25,15.0,0.0,equal_H15 | |
| 1187 | +real estate appraisal,A3,0.5,15.0,0.0,equal_H15 | |
| 1188 | +real estate appraisal,A3,0.75,15.0,0.6666666666666666,equal_H15 | |
| 1189 | +real estate appraisal,A3,1.0,15.0,1.0,equal_H15 | |
| 1190 | +real estate appraisal,A3,1.5,15.0,1.0,equal_H15 | |
| 1191 | +real estate appraisal,A3,2.0,15.0,1.0,equal_H15 | |
| 1192 | +real estate appraisal,A3,3.0,15.0,1.0,equal_H15 | |
| 1193 | +real estate appraisal,A3,4.0,15.0,1.0,equal_H15 | |
| 1194 | +real estate appraisal,A4,0.25,15.0,0.0,equal_H15 | |
| 1195 | +real estate appraisal,A4,0.5,15.0,0.0,equal_H15 | |
| 1196 | +real estate appraisal,A4,0.75,15.0,0.6666666666666666,equal_H15 | |
| 1197 | +real estate appraisal,A4,1.0,15.0,0.6666666666666666,equal_H15 | |
| 1198 | +real estate appraisal,A4,1.5,15.0,0.6666666666666666,equal_H15 | |
| 1199 | +real estate appraisal,A4,2.0,15.0,0.6666666666666666,equal_H15 | |
| 1200 | +real estate appraisal,A4,3.0,15.0,0.6666666666666666,equal_H15 | |
| 1201 | +real estate appraisal,A4,4.0,15.0,0.6666666666666666,equal_H15 | |
added
analysis/tables/table6_q_sensitivity_pilot.csv
+26 −0
Large diff (26 lines) is not loaded by default to keep the page fast.Load diff
added
analysis/tables/table7_judges_pilot.csv
+3 −0
Large diff (3 lines) is not loaded by default to keep the page fast.Load diff
added
analysis/tables/table8_det_vs_judges_pilot.csv
+3 −0
Large diff (3 lines) is not loaded by default to keep the page fast.Load diff
added
analysis/tables/table9_model_sensitivity.csv
+4 −0
Large diff (4 lines) is not loaded by default to keep the page fast.Load diff
added
analysis/tables/table9b_model_contrasts.csv
+9 −0
Large diff (9 lines) is not loaded by default to keep the page fast.Load diff
added
analysis/tables/table9c_model_by_scenario.csv
+31 −0
Large diff (31 lines) is not loaded by default to keep the page fast.Load diff
modified
dashboard/index.html
+1 −1
Large diff (4 lines) is not loaded by default to keep the page fast.Load diff
modified
data/raw/wp42.sqlite
+0 −0
Binary file not shown.
added
data/validated/events_pilot.parquet
+0 −0
Binary file not shown.
added
data/validated/events_recovery.parquet
+0 −0
Binary file not shown.
added
data/validated/events_sensitivity.parquet
+0 −0
Binary file not shown.
added
data/validated/judgments_pilot.csv
+451 −0
Large diff (451 lines) is not loaded by default to keep the page fast.Load diff
added
data/validated/judgments_pilot.parquet
+0 −0
Binary file not shown.
added
data/validated/judgments_recovery.csv
+121 −0
Large diff (121 lines) is not loaded by default to keep the page fast.Load diff
added
data/validated/judgments_recovery.parquet
+0 −0
Binary file not shown.
added
data/validated/judgments_sensitivity.csv
+121 −0
Large diff (121 lines) is not loaded by default to keep the page fast.Load diff
added
data/validated/judgments_sensitivity.parquet
+0 −0
Binary file not shown.
added
data/validated/receipts_pilot.csv
+151 −0
Large diff (151 lines) is not loaded by default to keep the page fast.Load diff
added
data/validated/receipts_pilot.parquet
+0 −0
Binary file not shown.
added
data/validated/receipts_recovery.csv
+41 −0
Large diff (41 lines) is not loaded by default to keep the page fast.Load diff
added
data/validated/receipts_recovery.parquet
+0 −0
Binary file not shown.
added
data/validated/receipts_sensitivity.csv
+41 −0
Large diff (41 lines) is not loaded by default to keep the page fast.Load diff
added
data/validated/receipts_sensitivity.parquet
+0 −0
Binary file not shown.
added
data/validated/runs_pilot.csv
+151 −0
Large diff (151 lines) is not loaded by default to keep the page fast.Load diff
added
data/validated/runs_pilot.parquet
+0 −0
Binary file not shown.
added
data/validated/runs_recovery.csv
+41 −0
Large diff (41 lines) is not loaded by default to keep the page fast.Load diff
added
data/validated/runs_recovery.parquet
+0 −0
Binary file not shown.
added
data/validated/runs_sensitivity.csv
+41 −0
Large diff (41 lines) is not loaded by default to keep the page fast.Load diff
added
data/validated/runs_sensitivity.parquet
+0 −0
Binary file not shown.
added
docs/OPERATIONS.md
+36 −0
Large diff (36 lines) is not loaded by default to keep the page fast.Load diff
added
docs/PHASE_5_AUDIT.md
+29 −0
Large diff (29 lines) is not loaded by default to keep the page fast.Load diff
added
logs/audit_pilot.txt
+46 −0
Large diff (46 lines) is not loaded by default to keep the page fast.Load diff
added
logs/dashboard.log
+0 −0
Large diff (0 lines) is not loaded by default to keep the page fast.Load diff
added
logs/estimate_pilot.txt
+53 −0
Large diff (53 lines) is not loaded by default to keep the page fast.Load diff
added
logs/pilot_run.log
+310 −0
Large diff (310 lines) is not loaded by default to keep the page fast.Load diff
added
logs/recovery.txt
+23 −0
Large diff (23 lines) is not loaded by default to keep the page fast.Load diff
added
logs/sensitivity.txt
+13 −0
Large diff (13 lines) is not loaded by default to keep the page fast.Load diff
added
logs/stage2_run.log
+117 −0
Large diff (117 lines) is not loaded by default to keep the page fast.Load diff
added
logs/stage3_run.log
+0 −0
Large diff (0 lines) is not loaded by default to keep the page fast.Load diff
added
logs/stage3b_run.log
+29 −0
Large diff (29 lines) is not loaded by default to keep the page fast.Load diff
modified
protocol/amendments.csv
+5 −0
Large diff (8 lines) is not loaded by default to keep the page fast.Load diff
modified
src/wp42/bundle.py
+4 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff
modified
src/wp42/cli.py
+2 −2
Large diff (16 lines) is not loaded by default to keep the page fast.Load diff
modified
src/wp42/judge.py
+10 −2
Large diff (29 lines) is not loaded by default to keep the page fast.Load diff
modified
src/wp42/launcher.py
+21 −8
Large diff (48 lines) is not loaded by default to keep the page fast.Load diff
modified
src/wp42/metrics.py
+1 −1
Large diff (8 lines) is not loaded by default to keep the page fast.Load diff
modified
src/wp42/scheduler.py
+5 −4
Large diff (33 lines) is not loaded by default to keep the page fast.Load diff
modified
src/wp42/store.py
+3 −0
Large diff (10 lines) is not loaded by default to keep the page fast.Load diff