1{2 "protocol": "heldout_v1",3 "generated": {4 "timestamp": "2026-11-04T00:02:26+00:00",5 "code_version": "d1aa8dc-dirty",6 "host": {7 "platform": "macOS-27.0-arm64-arm-64bit",8 "python": "3.12.13",9 "machine": "arm64",10 "node": "MacBookPro-de-simon-pierre-2.local"11 }12 },13 "n_runs": 199,14 "n_runs_ok": 199,15 "n_judgments": 33,16 "hypotheses": {17 "H1_D_vs_B": {18 "clarity": {19 "n": 24,20 "mean_diff": 0.042,21 "ci95": [22 -0.406,23 0.28124 ],25 "wilcoxon_p": 0.7920689965469446,26 "cliffs_delta": 0.014,27 "holm_p": 1.0,28 "significant": false29 },30 "argument_logic": {31 "n": 24,32 "mean_diff": 0.333,33 "ci95": [34 -0.219,35 0.65636 ],37 "wilcoxon_p": 0.1120871657779208,38 "cliffs_delta": 0.203,39 "holm_p": 0.4483,40 "significant": false41 },42 "contribution_positioning": {43 "n": 24,44 "mean_diff": 0.083,45 "ci95": [46 -0.406,47 0.43848 ],49 "wilcoxon_p": 0.9665165964249498,50 "cliffs_delta": 0.023,51 "holm_p": 0.9665,52 "significant": false53 },54 "usefulness": {55 "n": 24,56 "mean_diff": 0.167,57 "ci95": [58 -0.5,59 0.68860 ],61 "wilcoxon_p": 0.5747134367649175,62 "cliffs_delta": 0.007,63 "holm_p": 1.0,64 "significant": false65 }66 },67 "H2_D_vs_C": {68 "clarity": {69 "n": 24,70 "mean_diff": 0.0,71 "ci95": [72 -0.281,73 0.2574 ],75 "wilcoxon_p": 1.0,76 "cliffs_delta": 0.005,77 "holm_p": 1.0,78 "significant": false79 },80 "argument_logic": {81 "n": 24,82 "mean_diff": 0.083,83 "ci95": [84 -0.438,85 0.46986 ],87 "wilcoxon_p": 0.7506546409575028,88 "cliffs_delta": 0.0,89 "holm_p": 1.0,90 "significant": false91 },92 "contribution_positioning": {93 "n": 24,94 "mean_diff": -0.208,95 "ci95": [96 -0.719,97 0.15698 ],99 "wilcoxon_p": 0.15271894732542726,100 "cliffs_delta": -0.13,101 "holm_p": 0.6109,102 "significant": false103 },104 "usefulness": {105 "n": 24,106 "mean_diff": 0.042,107 "ci95": [108 -0.5,109 0.438110 ],111 "wilcoxon_p": 0.9426906297302409,112 "cliffs_delta": -0.068,113 "holm_p": 1.0,114 "significant": false115 }116 },117 "H3_E_vs_D": {118 "methodological_rigor": {119 "n": 24,120 "mean_diff": 1.333,121 "ci95": [122 1.0,123 1.845124 ],125 "wilcoxon_p": 9.280186701164916e-05,126 "cliffs_delta": 0.677,127 "holm_p": 0.0003,128 "significant": true129 },130 "fidelity": {131 "n": 24,132 "mean_diff": 0.625,133 "ci95": [134 0.312,135 1.031136 ],137 "wilcoxon_p": 0.0019691496770990976,138 "cliffs_delta": 0.359,139 "holm_p": 0.0039,140 "significant": true141 },142 "usefulness": {143 "n": 24,144 "mean_diff": 0.25,145 "ci95": [146 -0.031,147 0.656148 ],149 "wilcoxon_p": 0.10880943004054569,150 "cliffs_delta": 0.167,151 "holm_p": 0.1088,152 "significant": false153 }154 },155 "E_vs_A": {156 "clarity": {157 "n": 24,158 "mean_diff": -0.042,159 "ci95": [160 -0.406,161 0.281162 ],163 "wilcoxon_p": 0.7815112949987133,164 "cliffs_delta": -0.059,165 "holm_p": 0.7815,166 "significant": false167 },168 "argument_logic": {169 "n": 24,170 "mean_diff": 0.917,171 "ci95": [172 0.406,173 1.438174 ],175 "wilcoxon_p": 0.0008241896037559798,176 "cliffs_delta": 0.516,177 "holm_p": 0.0041,178 "significant": true179 },180 "contribution_positioning": {181 "n": 24,182 "mean_diff": 0.5,183 "ci95": [184 0.125,185 0.907186 ],187 "wilcoxon_p": 0.018735985158564714,188 "cliffs_delta": 0.198,189 "holm_p": 0.0562,190 "significant": false191 },192 "methodological_rigor": {193 "n": 24,194 "mean_diff": 2.042,195 "ci95": [196 1.438,197 2.625198 ],199 "wilcoxon_p": 4.6023226073469e-05,200 "cliffs_delta": 0.847,201 "holm_p": 0.0003,202 "significant": true203 },204 "fidelity": {205 "n": 24,206 "mean_diff": 0.417,207 "ci95": [208 0.156,209 0.781210 ],211 "wilcoxon_p": 0.012419330651552268,212 "cliffs_delta": 0.281,213 "holm_p": 0.0497,214 "significant": true215 },216 "usefulness": {217 "n": 24,218 "mean_diff": 0.458,219 "ci95": [220 -0.094,221 1.031222 ],223 "wilcoxon_p": 0.08329941488171003,224 "cliffs_delta": 0.163,225 "holm_p": 0.1666,226 "significant": false227 }228 },229 "E_vs_B": {230 "clarity": {231 "n": 24,232 "mean_diff": -0.042,233 "ci95": [234 -0.406,235 0.25236 ],237 "wilcoxon_p": 0.8127423725657857,238 "cliffs_delta": -0.05,239 "holm_p": 0.8127,240 "significant": false241 },242 "argument_logic": {243 "n": 24,244 "mean_diff": 0.792,245 "ci95": [246 0.406,247 1.062248 ],249 "wilcoxon_p": 0.0002901870694228414,250 "cliffs_delta": 0.483,251 "holm_p": 0.0015,252 "significant": true253 },254 "contribution_positioning": {255 "n": 24,256 "mean_diff": 0.25,257 "ci95": [258 -0.125,259 0.688260 ],261 "wilcoxon_p": 0.2720394478957354,262 "cliffs_delta": 0.089,263 "holm_p": 0.5441,264 "significant": false265 },266 "methodological_rigor": {267 "n": 24,268 "mean_diff": 1.667,269 "ci95": [270 1.125,271 2.188272 ],273 "wilcoxon_p": 8.456524858492154e-05,274 "cliffs_delta": 0.741,275 "holm_p": 0.0005,276 "significant": true277 },278 "fidelity": {279 "n": 24,280 "mean_diff": 1.042,281 "ci95": [282 0.781,283 1.438284 ],285 "wilcoxon_p": 0.0003061844202490992,286 "cliffs_delta": 0.573,287 "holm_p": 0.0012,288 "significant": true289 },290 "usefulness": {291 "n": 24,292 "mean_diff": 0.417,293 "ci95": [294 -0.125,295 0.969296 ],297 "wilcoxon_p": 0.11972288596282951,298 "cliffs_delta": 0.149,299 "holm_p": 0.3592,300 "significant": false301 }302 }303 },304 "ablations": {305 "E_no_retrieval": {306 "clarity": {307 "n": 16,308 "mean_diff": -0.125,309 "ci95": [310 -0.375,311 0.125312 ],313 "wilcoxon_p": 0.31731050786291415,314 "cliffs_delta": -0.117315 },316 "argument_logic": {317 "n": 16,318 "mean_diff": 0.188,319 "ci95": [320 -0.062,321 0.438322 ],323 "wilcoxon_p": 0.17971249487899976,324 "cliffs_delta": 0.141325 },326 "contribution_positioning": {327 "n": 16,328 "mean_diff": -0.062,329 "ci95": [330 -0.25,331 0.125332 ],333 "wilcoxon_p": 0.5637028616507731,334 "cliffs_delta": -0.047335 },336 "methodological_rigor": {337 "n": 16,338 "mean_diff": 0.125,339 "ci95": [340 -0.125,341 0.375342 ],343 "wilcoxon_p": 0.4142161782425253,344 "cliffs_delta": 0.109345 },346 "fidelity": {347 "n": 16,348 "mean_diff": 0.062,349 "ci95": [350 -0.312,351 0.438352 ],353 "wilcoxon_p": 0.7388826803635273,354 "cliffs_delta": 0.051355 },356 "usefulness": {357 "n": 16,358 "mean_diff": 0.0,359 "ci95": [360 -0.25,361 0.25362 ],363 "wilcoxon_p": 1.0,364 "cliffs_delta": 0.02365 }366 },367 "E_no_profile": {368 "clarity": {369 "n": 16,370 "mean_diff": -0.188,371 "ci95": [372 -0.562,373 0.125374 ],375 "wilcoxon_p": 0.31731050786291415,376 "cliffs_delta": -0.121377 },378 "argument_logic": {379 "n": 16,380 "mean_diff": 0.062,381 "ci95": [382 -0.25,383 0.375384 ],385 "wilcoxon_p": 0.7054569861112734,386 "cliffs_delta": 0.008387 },388 "contribution_positioning": {389 "n": 16,390 "mean_diff": -0.125,391 "ci95": [392 -0.438,393 0.188394 ],395 "wilcoxon_p": 0.4142161782425253,396 "cliffs_delta": -0.07397 },398 "methodological_rigor": {399 "n": 16,400 "mean_diff": 0.062,401 "ci95": [402 -0.312,403 0.5404 ],405 "wilcoxon_p": 0.763024600552995,406 "cliffs_delta": 0.02407 },408 "fidelity": {409 "n": 16,410 "mean_diff": 0.0,411 "ci95": [412 -0.438,413 0.438414 ],415 "wilcoxon_p": 1.0,416 "cliffs_delta": 0.008417 },418 "usefulness": {419 "n": 16,420 "mean_diff": -0.062,421 "ci95": [422 -0.312,423 0.188424 ],425 "wilcoxon_p": 0.6547208460185769,426 "cliffs_delta": -0.031427 }428 },429 "E_no_referee": {430 "clarity": {431 "n": 16,432 "mean_diff": -0.125,433 "ci95": [434 -0.438,435 0.188436 ],437 "wilcoxon_p": 0.4142161782425253,438 "cliffs_delta": -0.062439 },440 "argument_logic": {441 "n": 16,442 "mean_diff": 0.688,443 "ci95": [444 0.312,445 1.125446 ],447 "wilcoxon_p": 0.00838632096179826,448 "cliffs_delta": 0.43449 },450 "contribution_positioning": {451 "n": 16,452 "mean_diff": 0.25,453 "ci95": [454 -0.062,455 0.562456 ],457 "wilcoxon_p": 0.15729920705028516,458 "cliffs_delta": 0.09459 },460 "methodological_rigor": {461 "n": 16,462 "mean_diff": 1.625,463 "ci95": [464 1.0,465 2.188466 ],467 "wilcoxon_p": 0.0011302166601564717,468 "cliffs_delta": 0.781469 },470 "fidelity": {471 "n": 16,472 "mean_diff": 0.5,473 "ci95": [474 0.125,475 0.814476 ],477 "wilcoxon_p": 0.02092133533779403,478 "cliffs_delta": 0.289479 },480 "usefulness": {481 "n": 16,482 "mean_diff": 0.438,483 "ci95": [484 0.0,485 0.938486 ],487 "wilcoxon_p": 0.08326451666355039,488 "cliffs_delta": 0.246489 }490 },491 "E_no_factcheck": {492 "clarity": {493 "n": 16,494 "mean_diff": 0.0,495 "ci95": [496 -0.438,497 0.438498 ],499 "wilcoxon_p": 1.0,500 "cliffs_delta": 0.027501 },502 "argument_logic": {503 "n": 16,504 "mean_diff": 0.125,505 "ci95": [506 -0.25,507 0.438508 ],509 "wilcoxon_p": 0.47950012218695337,510 "cliffs_delta": 0.094511 },512 "contribution_positioning": {513 "n": 16,514 "mean_diff": 0.125,515 "ci95": [516 -0.125,517 0.375518 ],519 "wilcoxon_p": 0.31731050786291415,520 "cliffs_delta": 0.066521 },522 "methodological_rigor": {523 "n": 16,524 "mean_diff": 0.25,525 "ci95": [526 0.0,527 0.5528 ],529 "wilcoxon_p": 0.10247043485974945,530 "cliffs_delta": 0.113531 },532 "fidelity": {533 "n": 16,534 "mean_diff": 0.0,535 "ci95": [536 -0.25,537 0.25538 ],539 "wilcoxon_p": 1.0,540 "cliffs_delta": -0.008541 },542 "usefulness": {543 "n": 16,544 "mean_diff": 0.125,545 "ci95": [546 -0.188,547 0.438548 ],549 "wilcoxon_p": 0.47950012218695337,550 "cliffs_delta": 0.102551 }552 }553 },554 "judge_means": {555 "A_baseline": {556 "clarity": 5.67,557 "argument_logic": 5.04,558 "contribution_positioning": 4.12,559 "methodological_rigor": 4.17,560 "fidelity": 5.83,561 "usefulness": 4.83,562 "n_invented_facts": 0.79,563 "n": 24564 },565 "B_strong_prompt": {566 "clarity": 5.67,567 "argument_logic": 5.17,568 "contribution_positioning": 4.38,569 "methodological_rigor": 4.54,570 "fidelity": 5.21,571 "usefulness": 4.88,572 "n_invented_facts": 1.54,573 "n": 24574 },575 "C_conventional_rag": {576 "clarity": 5.71,577 "argument_logic": 5.42,578 "contribution_positioning": 4.67,579 "methodological_rigor": 4.58,580 "fidelity": 5.29,581 "usefulness": 5.0,582 "n_invented_facts": 1.33,583 "n": 24584 },585 "D_profiles_retrieval": {586 "clarity": 5.71,587 "argument_logic": 5.5,588 "contribution_positioning": 4.46,589 "methodological_rigor": 4.88,590 "fidelity": 5.62,591 "usefulness": 5.04,592 "n_invented_facts": 1.08,593 "n": 24594 },595 "E_full_engine": {596 "clarity": 5.62,597 "argument_logic": 5.96,598 "contribution_positioning": 4.62,599 "methodological_rigor": 6.21,600 "fidelity": 6.25,601 "usefulness": 5.29,602 "n_invented_facts": 0.42,603 "n": 24604 },605 "E_no_factcheck": {606 "clarity": 5.56,607 "argument_logic": 5.81,608 "contribution_positioning": 4.44,609 "methodological_rigor": 5.94,610 "fidelity": 6.12,611 "usefulness": 5.19,612 "n_invented_facts": 0.62,613 "n": 16614 },615 "E_no_profile": {616 "clarity": 5.75,617 "argument_logic": 5.88,618 "contribution_positioning": 4.69,619 "methodological_rigor": 6.12,620 "fidelity": 6.12,621 "usefulness": 5.38,622 "n_invented_facts": 0.69,623 "n": 16624 },625 "E_no_referee": {626 "clarity": 5.69,627 "argument_logic": 5.25,628 "contribution_positioning": 4.31,629 "methodological_rigor": 4.56,630 "fidelity": 5.62,631 "usefulness": 4.88,632 "n_invented_facts": 1.19,633 "n": 16634 },635 "E_no_retrieval": {636 "clarity": 5.69,637 "argument_logic": 5.75,638 "contribution_positioning": 4.62,639 "methodological_rigor": 6.06,640 "fidelity": 6.06,641 "usefulness": 5.31,642 "n_invented_facts": 0.69,643 "n": 16644 }645 },646 "task_means": {647 "('draft_introduction', 'A_baseline')": {648 "clarity": 6.25,649 "argument_logic": 5.75,650 "contribution_positioning": 5.0,651 "methodological_rigor": 4.25,652 "fidelity": 5.5,653 "usefulness": 5.75654 },655 "('draft_introduction', 'B_strong_prompt')": {656 "clarity": 6.0,657 "argument_logic": 5.5,658 "contribution_positioning": 5.0,659 "methodological_rigor": 4.25,660 "fidelity": 5.25,661 "usefulness": 5.5662 },663 "('draft_introduction', 'C_conventional_rag')": {664 "clarity": 6.0,665 "argument_logic": 5.75,666 "contribution_positioning": 5.5,667 "methodological_rigor": 4.75,668 "fidelity": 5.0,669 "usefulness": 5.5670 },671 "('draft_introduction', 'D_profiles_retrieval')": {672 "clarity": 6.0,673 "argument_logic": 6.0,674 "contribution_positioning": 5.0,675 "methodological_rigor": 5.25,676 "fidelity": 4.75,677 "usefulness": 5.5678 },679 "('draft_introduction', 'E_full_engine')": {680 "clarity": 5.75,681 "argument_logic": 6.5,682 "contribution_positioning": 5.0,683 "methodological_rigor": 6.5,684 "fidelity": 6.0,685 "usefulness": 6.0686 },687 "('draft_introduction', 'E_no_factcheck')": {688 "clarity": 5.5,689 "argument_logic": 5.5,690 "contribution_positioning": 5.0,691 "methodological_rigor": 6.0,692 "fidelity": 5.5,693 "usefulness": 5.5694 },695 "('draft_introduction', 'E_no_profile')": {696 "clarity": 5.5,697 "argument_logic": 6.0,698 "contribution_positioning": 4.5,699 "methodological_rigor": 6.0,700 "fidelity": 6.5,701 "usefulness": 5.5702 },703 "('draft_introduction', 'E_no_referee')": {704 "clarity": 6.0,705 "argument_logic": 6.5,706 "contribution_positioning": 5.0,707 "methodological_rigor": 5.0,708 "fidelity": 4.5,709 "usefulness": 5.0710 },711 "('draft_introduction', 'E_no_retrieval')": {712 "clarity": 6.0,713 "argument_logic": 6.0,714 "contribution_positioning": 5.0,715 "methodological_rigor": 7.0,716 "fidelity": 5.5,717 "usefulness": 6.0718 },719 "('explain_specification', 'A_baseline')": {720 "clarity": 6.0,721 "argument_logic": 5.0,722 "contribution_positioning": 3.5,723 "methodological_rigor": 4.5,724 "fidelity": 5.5,725 "usefulness": 4.5726 },727 "('explain_specification', 'B_strong_prompt')": {728 "clarity": 6.0,729 "argument_logic": 5.0,730 "contribution_positioning": 3.5,731 "methodological_rigor": 4.5,732 "fidelity": 5.5,733 "usefulness": 4.5734 },735 "('explain_specification', 'C_conventional_rag')": {736 "clarity": 5.5,737 "argument_logic": 5.0,738 "contribution_positioning": 4.5,739 "methodological_rigor": 4.0,740 "fidelity": 5.0,741 "usefulness": 4.5742 },743 "('explain_specification', 'D_profiles_retrieval')": {744 "clarity": 6.0,745 "argument_logic": 5.5,746 "contribution_positioning": 3.5,747 "methodological_rigor": 4.5,748 "fidelity": 5.5,749 "usefulness": 4.5750 },751 "('explain_specification', 'E_full_engine')": {752 "clarity": 5.5,753 "argument_logic": 5.5,754 "contribution_positioning": 3.5,755 "methodological_rigor": 6.0,756 "fidelity": 6.5,757 "usefulness": 5.0758 },759 "('explain_specification', 'E_no_factcheck')": {760 "clarity": 6.5,761 "argument_logic": 6.0,762 "contribution_positioning": 3.5,763 "methodological_rigor": 5.0,764 "fidelity": 6.0,765 "usefulness": 5.0766 },767 "('explain_specification', 'E_no_profile')": {768 "clarity": 6.0,769 "argument_logic": 6.0,770 "contribution_positioning": 3.5,771 "methodological_rigor": 6.0,772 "fidelity": 5.0,773 "usefulness": 5.5774 },775 "('explain_specification', 'E_no_referee')": {776 "clarity": 6.0,777 "argument_logic": 5.0,778 "contribution_positioning": 3.5,779 "methodological_rigor": 4.5,780 "fidelity": 6.0,781 "usefulness": 5.0782 },783 "('explain_specification', 'E_no_retrieval')": {784 "clarity": 6.0,785 "argument_logic": 5.5,786 "contribution_positioning": 3.5,787 "methodological_rigor": 5.5,788 "fidelity": 5.5,789 "usefulness": 5.0790 },791 "('flag_causal_claims', 'A_baseline')": {792 "clarity": 5.5,793 "argument_logic": 5.0,794 "contribution_positioning": 3.5,795 "methodological_rigor": 5.0,796 "fidelity": 6.0,797 "usefulness": 4.5798 },799 "('flag_causal_claims', 'B_strong_prompt')": {800 "clarity": 6.0,801 "argument_logic": 6.0,802 "contribution_positioning": 4.0,803 "methodological_rigor": 6.0,804 "fidelity": 5.5,805 "usefulness": 5.5806 },807 "('flag_causal_claims', 'C_conventional_rag')": {808 "clarity": 6.0,809 "argument_logic": 6.0,810 "contribution_positioning": 4.5,811 "methodological_rigor": 6.0,812 "fidelity": 6.0,813 "usefulness": 5.0814 },815 "('flag_causal_claims', 'D_profiles_retrieval')": {816 "clarity": 5.5,817 "argument_logic": 5.0,818 "contribution_positioning": 3.5,819 "methodological_rigor": 5.0,820 "fidelity": 6.0,821 "usefulness": 4.0822 },823 "('flag_causal_claims', 'E_full_engine')": {824 "clarity": 6.0,825 "argument_logic": 6.5,826 "contribution_positioning": 4.0,827 "methodological_rigor": 6.5,828 "fidelity": 6.5,829 "usefulness": 5.0830 },831 "('flag_causal_claims', 'E_no_factcheck')": {832 "clarity": 5.5,833 "argument_logic": 6.0,834 "contribution_positioning": 4.0,835 "methodological_rigor": 6.0,836 "fidelity": 6.0,837 "usefulness": 5.0838 },839 "('flag_causal_claims', 'E_no_profile')": {840 "clarity": 6.0,841 "argument_logic": 6.0,842 "contribution_positioning": 5.0,843 "methodological_rigor": 6.5,844 "fidelity": 6.5,845 "usefulness": 5.0846 },847 "('flag_causal_claims', 'E_no_referee')": {848 "clarity": 5.5,849 "argument_logic": 5.5,850 "contribution_positioning": 4.0,851 "methodological_rigor": 6.0,852 "fidelity": 6.0,853 "usefulness": 4.5854 },855 "('flag_causal_claims', 'E_no_retrieval')": {856 "clarity": 6.0,857 "argument_logic": 6.0,858 "contribution_positioning": 4.0,859 "methodological_rigor": 6.0,860 "fidelity": 6.5,861 "usefulness": 5.0862 },863 "('improve_abstract', 'A_baseline')": {864 "clarity": 5.0,865 "argument_logic": 5.25,866 "contribution_positioning": 4.5,867 "methodological_rigor": 4.0,868 "fidelity": 6.25,869 "usefulness": 4.75870 },871 "('improve_abstract', 'B_strong_prompt')": {872 "clarity": 5.0,873 "argument_logic": 5.0,874 "contribution_positioning": 5.0,875 "methodological_rigor": 4.25,876 "fidelity": 5.75,877 "usefulness": 4.75878 },879 "('improve_abstract', 'C_conventional_rag')": {880 "clarity": 5.5,881 "argument_logic": 5.5,882 "contribution_positioning": 5.25,883 "methodological_rigor": 4.25,884 "fidelity": 6.0,885 "usefulness": 5.25886 },887 "('improve_abstract', 'D_profiles_retrieval')": {888 "clarity": 5.25,889 "argument_logic": 5.25,890 "contribution_positioning": 4.75,891 "methodological_rigor": 4.25,892 "fidelity": 6.25,893 "usefulness": 5.0894 },895 "('improve_abstract', 'E_full_engine')": {896 "clarity": 5.0,897 "argument_logic": 5.25,898 "contribution_positioning": 4.75,899 "methodological_rigor": 5.25,900 "fidelity": 6.75,901 "usefulness": 4.75902 },903 "('improve_abstract', 'E_no_factcheck')": {904 "clarity": 5.0,905 "argument_logic": 6.0,906 "contribution_positioning": 5.0,907 "methodological_rigor": 5.0,908 "fidelity": 6.5,909 "usefulness": 5.0910 },911 "('improve_abstract', 'E_no_profile')": {912 "clarity": 5.5,913 "argument_logic": 5.5,914 "contribution_positioning": 5.0,915 "methodological_rigor": 5.0,916 "fidelity": 6.0,917 "usefulness": 5.0918 },919 "('improve_abstract', 'E_no_referee')": {920 "clarity": 5.0,921 "argument_logic": 5.0,922 "contribution_positioning": 4.5,923 "methodological_rigor": 4.0,924 "fidelity": 6.5,925 "usefulness": 5.0926 },927 "('improve_abstract', 'E_no_retrieval')": {928 "clarity": 4.5,929 "argument_logic": 5.5,930 "contribution_positioning": 5.0,931 "methodological_rigor": 5.5,932 "fidelity": 6.5,933 "usefulness": 5.0934 },935 "('improve_table', 'A_baseline')": {936 "clarity": 5.5,937 "argument_logic": 5.0,938 "contribution_positioning": 4.0,939 "methodological_rigor": 4.0,940 "fidelity": 6.0,941 "usefulness": 5.5942 },943 "('improve_table', 'B_strong_prompt')": {944 "clarity": 5.5,945 "argument_logic": 5.0,946 "contribution_positioning": 4.0,947 "methodological_rigor": 4.0,948 "fidelity": 5.5,949 "usefulness": 5.0950 },951 "('improve_table', 'C_conventional_rag')": {952 "clarity": 5.5,953 "argument_logic": 4.5,954 "contribution_positioning": 3.5,955 "methodological_rigor": 4.0,956 "fidelity": 5.0,957 "usefulness": 5.0958 },959 "('improve_table', 'D_profiles_retrieval')": {960 "clarity": 5.0,961 "argument_logic": 4.5,962 "contribution_positioning": 3.5,963 "methodological_rigor": 4.5,964 "fidelity": 5.5,965 "usefulness": 5.0966 },967 "('improve_table', 'E_full_engine')": {968 "clarity": 5.5,969 "argument_logic": 5.5,970 "contribution_positioning": 4.5,971 "methodological_rigor": 6.0,972 "fidelity": 6.5,973 "usefulness": 5.5974 },975 "('improve_table', 'E_no_factcheck')": {976 "clarity": 4.5,977 "argument_logic": 5.5,978 "contribution_positioning": 3.5,979 "methodological_rigor": 6.0,980 "fidelity": 6.5,981 "usefulness": 4.5982 },983 "('improve_table', 'E_no_profile')": {984 "clarity": 5.0,985 "argument_logic": 5.5,986 "contribution_positioning": 4.0,987 "methodological_rigor": 6.5,988 "fidelity": 6.5,989 "usefulness": 5.0990 },991 "('improve_table', 'E_no_referee')": {992 "clarity": 6.0,993 "argument_logic": 5.0,994 "contribution_positioning": 4.0,995 "methodological_rigor": 4.5,996 "fidelity": 5.5,997 "usefulness": 6.0998 },999 "('improve_table', 'E_no_retrieval')": {1000 "clarity": 5.5,1001 "argument_logic": 5.5,1002 "contribution_positioning": 4.5,1003 "methodological_rigor": 6.0,1004 "fidelity": 6.0,1005 "usefulness": 5.01006 },1007 "('interpret_results', 'A_baseline')": {1008 "clarity": 6.0,1009 "argument_logic": 5.0,1010 "contribution_positioning": 4.0,1011 "methodological_rigor": 4.5,1012 "fidelity": 6.0,1013 "usefulness": 5.01014 },1015 "('interpret_results', 'B_strong_prompt')": {1016 "clarity": 6.0,1017 "argument_logic": 5.25,1018 "contribution_positioning": 4.5,1019 "methodological_rigor": 5.0,1020 "fidelity": 5.0,1021 "usefulness": 5.01022 },1023 "('interpret_results', 'C_conventional_rag')": {1024 "clarity": 6.0,1025 "argument_logic": 6.0,1026 "contribution_positioning": 4.5,1027 "methodological_rigor": 5.25,1028 "fidelity": 5.0,1029 "usefulness": 5.251030 },1031 "('interpret_results', 'D_profiles_retrieval')": {1032 "clarity": 6.0,1033 "argument_logic": 5.5,1034 "contribution_positioning": 4.25,1035 "methodological_rigor": 5.0,1036 "fidelity": 5.5,1037 "usefulness": 5.01038 },1039 "('interpret_results', 'E_full_engine')": {1040 "clarity": 5.75,1041 "argument_logic": 6.25,1042 "contribution_positioning": 4.5,1043 "methodological_rigor": 6.75,1044 "fidelity": 5.75,1045 "usefulness": 5.01046 },1047 "('interpret_results', 'E_no_factcheck')": {1048 "clarity": 6.0,1049 "argument_logic": 6.0,1050 "contribution_positioning": 4.5,1051 "methodological_rigor": 7.0,1052 "fidelity": 6.5,1053 "usefulness": 5.51054 },1055 "('interpret_results', 'E_no_profile')": {1056 "clarity": 6.0,1057 "argument_logic": 6.0,1058 "contribution_positioning": 4.5,1059 "methodological_rigor": 6.0,1060 "fidelity": 6.5,1061 "usefulness": 5.51062 },1063 "('interpret_results', 'E_no_referee')": {1064 "clarity": 6.0,1065 "argument_logic": 5.0,1066 "contribution_positioning": 4.5,1067 "methodological_rigor": 5.0,1068 "fidelity": 6.0,1069 "usefulness": 5.01070 },1071 "('interpret_results', 'E_no_retrieval')": {1072 "clarity": 6.0,1073 "argument_logic": 6.0,1074 "contribution_positioning": 4.5,1075 "methodological_rigor": 6.5,1076 "fidelity": 6.5,1077 "usefulness": 5.51078 },1079 "('referee_report', 'A_baseline')": {1080 "clarity": 5.25,1081 "argument_logic": 4.25,1082 "contribution_positioning": 3.25,1083 "methodological_rigor": 3.75,1084 "fidelity": 6.0,1085 "usefulness": 3.751086 },1087 "('referee_report', 'B_strong_prompt')": {1088 "clarity": 5.25,1089 "argument_logic": 4.75,1090 "contribution_positioning": 3.25,1091 "methodological_rigor": 4.5,1092 "fidelity": 5.25,1093 "usefulness": 4.01094 },1095 "('referee_report', 'C_conventional_rag')": {1096 "clarity": 5.25,1097 "argument_logic": 4.5,1098 "contribution_positioning": 3.5,1099 "methodological_rigor": 4.0,1100 "fidelity": 5.25,1101 "usefulness": 3.751102 },1103 "('referee_report', 'D_profiles_retrieval')": {1104 "clarity": 5.75,1105 "argument_logic": 6.0,1106 "contribution_positioning": 4.75,1107 "methodological_rigor": 5.75,1108 "fidelity": 6.25,1109 "usefulness": 5.251110 },1111 "('referee_report', 'E_full_engine')": {1112 "clarity": 6.0,1113 "argument_logic": 6.25,1114 "contribution_positioning": 4.75,1115 "methodological_rigor": 6.25,1116 "fidelity": 6.5,1117 "usefulness": 5.51118 },1119 "('referee_report', 'E_no_factcheck')": {1120 "clarity": 5.5,1121 "argument_logic": 5.5,1122 "contribution_positioning": 4.0,1123 "methodological_rigor": 6.0,1124 "fidelity": 6.5,1125 "usefulness": 5.01126 },1127 "('referee_report', 'E_no_profile')": {1128 "clarity": 6.0,1129 "argument_logic": 6.0,1130 "contribution_positioning": 5.0,1131 "methodological_rigor": 6.5,1132 "fidelity": 6.0,1133 "usefulness": 5.51134 },1135 "('referee_report', 'E_no_referee')": {1136 "clarity": 5.5,1137 "argument_logic": 4.5,1138 "contribution_positioning": 3.5,1139 "methodological_rigor": 4.0,1140 "fidelity": 6.0,1141 "usefulness": 3.51142 },1143 "('referee_report', 'E_no_retrieval')": {1144 "clarity": 6.0,1145 "argument_logic": 6.0,1146 "contribution_positioning": 5.0,1147 "methodological_rigor": 6.0,1148 "fidelity": 6.5,1149 "usefulness": 5.51150 },1151 "('rewrite_literature', 'A_baseline')": {1152 "clarity": 6.0,1153 "argument_logic": 5.0,1154 "contribution_positioning": 5.0,1155 "methodological_rigor": 3.5,1156 "fidelity": 5.0,1157 "usefulness": 5.01158 },1159 "('rewrite_literature', 'B_strong_prompt')": {1160 "clarity": 6.0,1161 "argument_logic": 5.0,1162 "contribution_positioning": 5.5,1163 "methodological_rigor": 4.0,1164 "fidelity": 3.5,1165 "usefulness": 5.01166 },1167 "('rewrite_literature', 'C_conventional_rag')": {1168 "clarity": 6.0,1169 "argument_logic": 6.0,1170 "contribution_positioning": 6.0,1171 "methodological_rigor": 4.5,1172 "fidelity": 5.0,1173 "usefulness": 6.01174 },1175 "('rewrite_literature', 'D_profiles_retrieval')": {1176 "clarity": 6.0,1177 "argument_logic": 5.5,1178 "contribution_positioning": 5.5,1179 "methodological_rigor": 4.0,1180 "fidelity": 5.0,1181 "usefulness": 5.51182 },1183 "('rewrite_literature', 'E_full_engine')": {1184 "clarity": 5.5,1185 "argument_logic": 5.5,1186 "contribution_positioning": 5.5,1187 "methodological_rigor": 6.5,1188 "fidelity": 5.5,1189 "usefulness": 5.51190 },1191 "('rewrite_literature', 'E_no_factcheck')": {1192 "clarity": 6.0,1193 "argument_logic": 6.0,1194 "contribution_positioning": 6.0,1195 "methodological_rigor": 6.5,1196 "fidelity": 5.5,1197 "usefulness": 6.01198 },1199 "('rewrite_literature', 'E_no_profile')": {1200 "clarity": 6.0,1201 "argument_logic": 6.0,1202 "contribution_positioning": 6.0,1203 "methodological_rigor": 6.5,1204 "fidelity": 6.0,1205 "usefulness": 6.01206 },1207 "('rewrite_literature', 'E_no_referee')": {1208 "clarity": 5.5,1209 "argument_logic": 5.5,1210 "contribution_positioning": 5.5,1211 "methodological_rigor": 3.5,1212 "fidelity": 4.5,1213 "usefulness": 5.01214 },1215 "('rewrite_literature', 'E_no_retrieval')": {1216 "clarity": 5.5,1217 "argument_logic": 5.5,1218 "contribution_positioning": 5.5,1219 "methodological_rigor": 6.0,1220 "fidelity": 5.5,1221 "usefulness": 5.51222 }1223 },1224 "out_of_domain_means": {1225 "A_baseline": {1226 "clarity": 5.67,1227 "argument_logic": 5.33,1228 "contribution_positioning": 3.67,1229 "methodological_rigor": 4.0,1230 "fidelity": 5.33,1231 "usefulness": 4.01232 },1233 "B_strong_prompt": {1234 "clarity": 5.67,1235 "argument_logic": 5.33,1236 "contribution_positioning": 4.0,1237 "methodological_rigor": 3.67,1238 "fidelity": 5.0,1239 "usefulness": 4.01240 },1241 "C_conventional_rag": {1242 "clarity": 6.33,1243 "argument_logic": 5.67,1244 "contribution_positioning": 5.33,1245 "methodological_rigor": 5.0,1246 "fidelity": 4.67,1247 "usefulness": 5.671248 },1249 "D_profiles_retrieval": {1250 "clarity": 5.33,1251 "argument_logic": 5.33,1252 "contribution_positioning": 4.0,1253 "methodological_rigor": 4.33,1254 "fidelity": 5.33,1255 "usefulness": 4.01256 },1257 "E_full_engine": {1258 "clarity": 5.33,1259 "argument_logic": 5.33,1260 "contribution_positioning": 3.67,1261 "methodological_rigor": 5.0,1262 "fidelity": 4.67,1263 "usefulness": 4.01264 }1265 },1266 "judge_consistency": {1267 "n_pairs": 50,1268 "clarity": {1269 "mean_abs_diff": 0.18,1270 "spearman": 0.7121271 },1272 "argument_logic": {1273 "mean_abs_diff": 0.38,1274 "spearman": 0.5991275 },1276 "contribution_positioning": {1277 "mean_abs_diff": 0.3,1278 "spearman": 0.9091279 },1280 "methodological_rigor": {1281 "mean_abs_diff": 0.28,1282 "spearman": 0.8931283 },1284 "fidelity": {1285 "mean_abs_diff": 0.44,1286 "spearman": 0.7391287 },1288 "usefulness": {1289 "mean_abs_diff": 0.24,1290 "spearman": 0.8181291 }1292 },1293 "secondary_means": {1294 "A_baseline": {1295 "preserved_rate": 0.199,1296 "new_numbers": 1.583,1297 "n_unverifiable_citations": 0.0,1298 "n_unsupported_claims": 1.167,1299 "max_overlap_share": 0.041,1300 "todo_placeholders": 12.292,1301 "n_words": 769.333,1302 "cost_usd": 0.076,1303 "latency_s": 33.675,1304 "n": 241305 },1306 "B_strong_prompt": {1307 "preserved_rate": 0.276,1308 "new_numbers": 1.208,1309 "n_unverifiable_citations": 0.0,1310 "n_unsupported_claims": 1.792,1311 "max_overlap_share": 0.043,1312 "todo_placeholders": 15.458,1313 "n_words": 828.417,1314 "cost_usd": 0.081,1315 "latency_s": 36.117,1316 "n": 241317 },1318 "C_conventional_rag": {1319 "preserved_rate": 0.244,1320 "new_numbers": 1.458,1321 "n_unverifiable_citations": 0.0,1322 "n_unsupported_claims": 1.5,1323 "max_overlap_share": 0.033,1324 "todo_placeholders": 12.625,1325 "n_words": 844.625,1326 "cost_usd": 0.087,1327 "latency_s": 37.613,1328 "n": 241329 },1330 "D_profiles_retrieval": {1331 "preserved_rate": 0.238,1332 "new_numbers": 1.833,1333 "n_unverifiable_citations": 0.0,1334 "n_unsupported_claims": 1.458,1335 "max_overlap_share": 0.032,1336 "todo_placeholders": 12.458,1337 "n_words": 847.5,1338 "cost_usd": 0.095,1339 "latency_s": 38.204,1340 "n": 241341 },1342 "E_full_engine": {1343 "preserved_rate": 0.302,1344 "new_numbers": 0.708,1345 "n_unverifiable_citations": 0.0,1346 "n_unsupported_claims": 0.875,1347 "max_overlap_share": 0.048,1348 "todo_placeholders": 12.25,1349 "n_words": 981.25,1350 "cost_usd": 0.202,1351 "latency_s": 80.062,1352 "n": 241353 },1354 "E_no_factcheck": {1355 "preserved_rate": 0.4,1356 "new_numbers": 1.938,1357 "n_unverifiable_citations": 0.0,1358 "n_unsupported_claims": 1.0,1359 "max_overlap_share": 0.025,1360 "todo_placeholders": 12.875,1361 "n_words": 909.062,1362 "cost_usd": 0.197,1363 "latency_s": 77.606,1364 "n": 161365 },1366 "E_no_profile": {1367 "preserved_rate": 0.364,1368 "new_numbers": 0.625,1369 "n_unverifiable_citations": 0.0,1370 "n_unsupported_claims": 1.125,1371 "max_overlap_share": 0.03,1372 "todo_placeholders": 13.062,1373 "n_words": 931.5,1374 "cost_usd": 0.194,1375 "latency_s": 77.194,1376 "n": 161377 },1378 "E_no_referee": {1379 "preserved_rate": 0.221,1380 "new_numbers": 1.438,1381 "n_unverifiable_citations": 0.0,1382 "n_unsupported_claims": 0.938,1383 "max_overlap_share": 0.041,1384 "todo_placeholders": 12.125,1385 "n_words": 790.812,1386 "cost_usd": 0.097,1387 "latency_s": 39.5,1388 "n": 161389 },1390 "E_no_retrieval": {1391 "preserved_rate": 0.305,1392 "new_numbers": 0.938,1393 "n_unverifiable_citations": 0.0,1394 "n_unsupported_claims": 0.75,1395 "max_overlap_share": 0.047,1396 "todo_placeholders": 14.312,1397 "n_words": 897.938,1398 "cost_usd": 0.195,1399 "latency_s": 78.019,1400 "n": 161401 }1402 },1403 "H4": {1404 "E_vs_A": {1405 "n_unverifiable_citations": {1406 "n": 24,1407 "mean_diff": 0.0,1408 "ci95": [1409 0.0,1410 0.01411 ],1412 "wilcoxon_p": 1.0,1413 "cliffs_delta": 0.01414 },1415 "n_unsupported_claims": {1416 "n": 24,1417 "mean_diff": -0.292,1418 "ci95": [1419 -0.875,1420 0.4691421 ],1422 "wilcoxon_p": 0.22484337530538745,1423 "cliffs_delta": -0.1091424 },1425 "new_numbers": {1426 "n": 24,1427 "mean_diff": -0.875,1428 "ci95": [1429 -1.938,1430 -0.0311431 ],1432 "wilcoxon_p": 0.040684908780744745,1433 "cliffs_delta": -0.1131434 }1435 },1436 "E_vs_B": {1437 "n_unverifiable_citations": {1438 "n": 24,1439 "mean_diff": 0.0,1440 "ci95": [1441 0.0,1442 0.01443 ],1444 "wilcoxon_p": 1.0,1445 "cliffs_delta": 0.01446 },1447 "n_unsupported_claims": {1448 "n": 24,1449 "mean_diff": -0.917,1450 "ci95": [1451 -1.688,1452 0.1251453 ],1454 "wilcoxon_p": 0.05044606501119172,1455 "cliffs_delta": -0.3121456 },1457 "new_numbers": {1458 "n": 24,1459 "mean_diff": -0.5,1460 "ci95": [1461 -1.125,1462 0.01463 ],1464 "wilcoxon_p": 0.08446903225325182,1465 "cliffs_delta": -0.0831466 }1467 },1468 "E_vs_C": {1469 "n_unverifiable_citations": {1470 "n": 24,1471 "mean_diff": 0.0,1472 "ci95": [1473 0.0,1474 0.01475 ],1476 "wilcoxon_p": 1.0,1477 "cliffs_delta": 0.01478 },1479 "n_unsupported_claims": {1480 "n": 24,1481 "mean_diff": -0.625,1482 "ci95": [1483 -1.281,1484 0.2191485 ],1486 "wilcoxon_p": 0.07156571124393864,1487 "cliffs_delta": -0.241488 },1489 "new_numbers": {1490 "n": 24,1491 "mean_diff": -0.75,1492 "ci95": [1493 -1.344,1494 -0.251495 ],1496 "wilcoxon_p": 0.026014479622894222,1497 "cliffs_delta": -0.1251498 }1499 }1500 },1501 "H5_overlap": {1502 "preregistered_metric": {1503 "max_overlap_share_overall": 0.36,1504 "share_outputs_below_0.02": 0.56521739130434781505 },1506 "excluding_source_document": {1507 "max_other_share": 0.0063,1508 "share_outputs_below_0.02": 1.0,1509 "mean_other_share": 0.00058315217391304361510 },1511 "self_overlap_by_task": {1512 "draft_introduction": 0.018,1513 "explain_specification": 0.012,1514 "flag_causal_claims": 0.038,1515 "improve_abstract": 0.117,1516 "improve_table": 0.035,1517 "interpret_results": 0.012,1518 "referee_report": 0.006,1519 "rewrite_literature": 0.0631520 },1521 "self_overlap_by_condition": {1522 "A_baseline": 0.04,1523 "B_strong_prompt": 0.043,1524 "C_conventional_rag": 0.033,1525 "D_profiles_retrieval": 0.032,1526 "E_full_engine": 0.048,1527 "E_no_factcheck": 0.025,1528 "E_no_profile": 0.03,1529 "E_no_referee": 0.041,1530 "E_no_retrieval": 0.0471531 }1532 }1533}