1{2 "protocol": "heldout_v1",3 "generated": {4 "timestamp": "2026-11-04T00:51:23+00:00",5 "code_version": "760cf48-dirty",6 "host": {7 "platform": "macOS-27.0-arm64-arm-64bit",8 "python": "3.12.13",9 "machine": "arm64",10 "node": "MacBookPro-de-simon-pierre-2.local"11 }12 },13 "n_runs": 199,14 "n_runs_ok": 199,15 "n_judgments": 33,16 "hypotheses": {17 "H1_D_vs_B": {18 "clarity": {19 "n": 24,20 "mean_diff": -0.125,21 "ci95": [22 -0.312,23 0.03124 ],25 "wilcoxon_p": 0.17971249487899976,26 "cliffs_delta": -0.118,27 "holm_p": 0.7188,28 "significant": false29 },30 "argument_logic": {31 "n": 24,32 "mean_diff": 0.042,33 "ci95": [34 -0.156,35 0.21936 ],37 "wilcoxon_p": 0.5637028616507731,38 "cliffs_delta": 0.04,39 "holm_p": 1.0,40 "significant": false41 },42 "contribution_positioning": {43 "n": 24,44 "mean_diff": 0.0,45 "ci95": [46 -0.281,47 0.28148 ],49 "wilcoxon_p": 1.0,50 "cliffs_delta": 0.014,51 "holm_p": 1.0,52 "significant": false53 },54 "usefulness": {55 "n": 24,56 "mean_diff": 0.042,57 "ci95": [58 -0.281,59 0.40660 ],61 "wilcoxon_p": 0.7815112949987133,62 "cliffs_delta": 0.038,63 "holm_p": 1.0,64 "significant": false65 }66 },67 "H2_D_vs_C": {68 "clarity": {69 "n": 24,70 "mean_diff": -0.042,71 "ci95": [72 -0.094,73 0.074 ],75 "wilcoxon_p": 0.31731050786291415,76 "cliffs_delta": -0.042,77 "holm_p": 0.9519,78 "significant": false79 },80 "argument_logic": {81 "n": 24,82 "mean_diff": -0.125,83 "ci95": [84 -0.219,85 0.03186 ],87 "wilcoxon_p": 0.17971249487899976,88 "cliffs_delta": -0.12,89 "holm_p": 0.7188,90 "significant": false91 },92 "contribution_positioning": {93 "n": 24,94 "mean_diff": 0.125,95 "ci95": [96 -0.031,97 0.21998 ],99 "wilcoxon_p": 0.31731050786291415,100 "cliffs_delta": 0.132,101 "holm_p": 0.6346,102 "significant": false103 },104 "usefulness": {105 "n": 24,106 "mean_diff": 0.083,107 "ci95": [108 -0.062,109 0.344110 ],111 "wilcoxon_p": 0.47950012218695337,112 "cliffs_delta": 0.069,113 "holm_p": 0.4795,114 "significant": false115 }116 },117 "H3_E_vs_D": {118 "methodological_rigor": {119 "n": 24,120 "mean_diff": 0.958,121 "ci95": [122 0.438,123 1.469124 ],125 "wilcoxon_p": 0.0028308913160744967,126 "cliffs_delta": 0.55,127 "holm_p": 0.0085,128 "significant": true129 },130 "fidelity": {131 "n": 24,132 "mean_diff": 0.458,133 "ci95": [134 0.031,135 0.688136 ],137 "wilcoxon_p": 0.011831666473446543,138 "cliffs_delta": 0.323,139 "holm_p": 0.0237,140 "significant": true141 },142 "usefulness": {143 "n": 24,144 "mean_diff": -0.333,145 "ci95": [146 -0.781,147 0.062148 ],149 "wilcoxon_p": 0.06462747314963886,150 "cliffs_delta": -0.243,151 "holm_p": 0.0646,152 "significant": false153 }154 },155 "E_vs_A": {156 "clarity": {157 "n": 24,158 "mean_diff": 0.042,159 "ci95": [160 -0.25,161 0.688162 ],163 "wilcoxon_p": 0.7388826803635273,164 "cliffs_delta": -0.111,165 "holm_p": 0.7389,166 "significant": false167 },168 "argument_logic": {169 "n": 24,170 "mean_diff": 0.458,171 "ci95": [172 0.094,173 1.313174 ],175 "wilcoxon_p": 0.11558942362652867,176 "cliffs_delta": 0.245,177 "holm_p": 0.4624,178 "significant": false179 },180 "contribution_positioning": {181 "n": 24,182 "mean_diff": -0.083,183 "ci95": [184 -0.375,185 0.719186 ],187 "wilcoxon_p": 0.4828996977801915,188 "cliffs_delta": -0.137,189 "holm_p": 0.9658,190 "significant": false191 },192 "methodological_rigor": {193 "n": 24,194 "mean_diff": 1.333,195 "ci95": [196 0.844,197 2.282198 ],199 "wilcoxon_p": 0.0003879158284341555,200 "cliffs_delta": 0.651,201 "holm_p": 0.0023,202 "significant": true203 },204 "fidelity": {205 "n": 24,206 "mean_diff": 0.375,207 "ci95": [208 -0.031,209 1.125210 ],211 "wilcoxon_p": 0.0577795711235972,212 "cliffs_delta": 0.22,213 "holm_p": 0.2889,214 "significant": false215 },216 "usefulness": {217 "n": 24,218 "mean_diff": -0.167,219 "ci95": [220 -0.562,221 0.438222 ],223 "wilcoxon_p": 0.3142753351370735,224 "cliffs_delta": -0.215,225 "holm_p": 0.9428,226 "significant": false227 }228 },229 "E_vs_B": {230 "clarity": {231 "n": 24,232 "mean_diff": -0.208,233 "ci95": [234 -0.469,235 -0.062236 ],237 "wilcoxon_p": 0.13166801602281422,238 "cliffs_delta": -0.193,239 "holm_p": 0.1317,240 "significant": false241 },242 "argument_logic": {243 "n": 24,244 "mean_diff": 0.333,245 "ci95": [246 0.094,247 0.688248 ],249 "wilcoxon_p": 0.08288063685529753,250 "cliffs_delta": 0.271,251 "holm_p": 0.2486,252 "significant": false253 },254 "contribution_positioning": {255 "n": 24,256 "mean_diff": -0.417,257 "ci95": [258 -0.75,259 0.094260 ],261 "wilcoxon_p": 0.07260252961103252,262 "cliffs_delta": -0.29,263 "holm_p": 0.2904,264 "significant": false265 },266 "methodological_rigor": {267 "n": 24,268 "mean_diff": 1.083,269 "ci95": [270 0.562,271 1.812272 ],273 "wilcoxon_p": 0.0014260915957594058,274 "cliffs_delta": 0.545,275 "holm_p": 0.0086,276 "significant": true277 },278 "fidelity": {279 "n": 24,280 "mean_diff": 0.375,281 "ci95": [282 0.156,283 0.594284 ],285 "wilcoxon_p": 0.012554918596966537,286 "cliffs_delta": 0.299,287 "holm_p": 0.0628,288 "significant": false289 },290 "usefulness": {291 "n": 24,292 "mean_diff": -0.292,293 "ci95": [294 -0.657,295 0.031296 ],297 "wilcoxon_p": 0.11709281778662677,298 "cliffs_delta": -0.207,299 "holm_p": 0.2342,300 "significant": false301 }302 }303 },304 "ablations": {305 "E_no_retrieval": {306 "clarity": {307 "n": 16,308 "mean_diff": 0.0,309 "ci95": [310 -0.25,311 0.312312 ],313 "wilcoxon_p": 1.0,314 "cliffs_delta": -0.008315 },316 "argument_logic": {317 "n": 16,318 "mean_diff": 0.188,319 "ci95": [320 -0.125,321 0.5322 ],323 "wilcoxon_p": 0.25683925795785656,324 "cliffs_delta": 0.176325 },326 "contribution_positioning": {327 "n": 16,328 "mean_diff": -0.25,329 "ci95": [330 -0.562,331 0.062332 ],333 "wilcoxon_p": 0.15729920705028505,334 "cliffs_delta": -0.125335 },336 "methodological_rigor": {337 "n": 16,338 "mean_diff": -0.062,339 "ci95": [340 -0.625,341 0.438342 ],343 "wilcoxon_p": 0.9416269114492879,344 "cliffs_delta": -0.02345 },346 "fidelity": {347 "n": 16,348 "mean_diff": 0.062,349 "ci95": [350 -0.188,351 0.375352 ],353 "wilcoxon_p": 0.6547208460185769,354 "cliffs_delta": 0.082355 },356 "usefulness": {357 "n": 16,358 "mean_diff": -0.125,359 "ci95": [360 -0.562,361 0.312362 ],363 "wilcoxon_p": 0.5574801348628696,364 "cliffs_delta": -0.047365 }366 },367 "E_no_profile": {368 "clarity": {369 "n": 16,370 "mean_diff": -0.062,371 "ci95": [372 -0.25,373 0.125374 ],375 "wilcoxon_p": 0.5637028616507731,376 "cliffs_delta": -0.066377 },378 "argument_logic": {379 "n": 16,380 "mean_diff": 0.0,381 "ci95": [382 -0.188,383 0.188384 ],385 "wilcoxon_p": 1.0,386 "cliffs_delta": 0.0387 },388 "contribution_positioning": {389 "n": 16,390 "mean_diff": -0.188,391 "ci95": [392 -0.375,393 0.0394 ],395 "wilcoxon_p": 0.08326451666355039,396 "cliffs_delta": -0.113397 },398 "methodological_rigor": {399 "n": 16,400 "mean_diff": -0.25,401 "ci95": [402 -0.5,403 0.0404 ],405 "wilcoxon_p": 0.10247043485974945,406 "cliffs_delta": -0.148407 },408 "fidelity": {409 "n": 16,410 "mean_diff": 0.0,411 "ci95": [412 -0.25,413 0.25414 ],415 "wilcoxon_p": 1.0,416 "cliffs_delta": -0.043417 },418 "usefulness": {419 "n": 16,420 "mean_diff": -0.25,421 "ci95": [422 -0.438,423 -0.062424 ],425 "wilcoxon_p": 0.045500263896358396,426 "cliffs_delta": -0.176427 }428 },429 "E_no_referee": {430 "clarity": {431 "n": 16,432 "mean_diff": -0.125,433 "ci95": [434 -0.375,435 0.125436 ],437 "wilcoxon_p": 0.31731050786291415,438 "cliffs_delta": -0.125439 },440 "argument_logic": {441 "n": 16,442 "mean_diff": 0.5,443 "ci95": [444 0.062,445 0.938446 ],447 "wilcoxon_p": 0.045579154787925666,448 "cliffs_delta": 0.422449 },450 "contribution_positioning": {451 "n": 16,452 "mean_diff": 0.062,453 "ci95": [454 -0.375,455 0.438456 ],457 "wilcoxon_p": 0.763024600552995,458 "cliffs_delta": 0.094459 },460 "methodological_rigor": {461 "n": 16,462 "mean_diff": 1.25,463 "ci95": [464 0.625,465 1.875466 ],467 "wilcoxon_p": 0.003243891789291054,468 "cliffs_delta": 0.625469 },470 "fidelity": {471 "n": 16,472 "mean_diff": 0.562,473 "ci95": [474 0.125,475 1.125476 ],477 "wilcoxon_p": 0.04734500208525526,478 "cliffs_delta": 0.371479 },480 "usefulness": {481 "n": 16,482 "mean_diff": 0.125,483 "ci95": [484 -0.375,485 0.562486 ],487 "wilcoxon_p": 0.5929800980174267,488 "cliffs_delta": 0.156489 }490 },491 "E_no_factcheck": {492 "clarity": {493 "n": 16,494 "mean_diff": -0.125,495 "ci95": [496 -0.375,497 0.125498 ],499 "wilcoxon_p": 0.31731050786291415,500 "cliffs_delta": -0.117501 },502 "argument_logic": {503 "n": 16,504 "mean_diff": 0.062,505 "ci95": [506 -0.314,507 0.438508 ],509 "wilcoxon_p": 0.7388826803635273,510 "cliffs_delta": 0.059511 },512 "contribution_positioning": {513 "n": 16,514 "mean_diff": 0.0,515 "ci95": [516 -0.312,517 0.312518 ],519 "wilcoxon_p": 1.0,520 "cliffs_delta": 0.086521 },522 "methodological_rigor": {523 "n": 16,524 "mean_diff": -0.062,525 "ci95": [526 -0.562,527 0.375528 ],529 "wilcoxon_p": 1.0,530 "cliffs_delta": 0.004531 },532 "fidelity": {533 "n": 16,534 "mean_diff": -0.062,535 "ci95": [536 -0.375,537 0.25538 ],539 "wilcoxon_p": 0.7054569861112734,540 "cliffs_delta": -0.062541 },542 "usefulness": {543 "n": 16,544 "mean_diff": -0.188,545 "ci95": [546 -0.5,547 0.125548 ],549 "wilcoxon_p": 0.25683925795785656,550 "cliffs_delta": -0.098551 }552 }553 },554 "judge_means": {555 "A_baseline": {556 "clarity": 5.79,557 "argument_logic": 5.58,558 "contribution_positioning": 5.0,559 "methodological_rigor": 4.79,560 "fidelity": 5.33,561 "usefulness": 5.58,562 "n_invented_facts": 1.38,563 "n": 24564 },565 "B_strong_prompt": {566 "clarity": 6.04,567 "argument_logic": 5.71,568 "contribution_positioning": 5.33,569 "methodological_rigor": 5.04,570 "fidelity": 5.33,571 "usefulness": 5.71,572 "n_invented_facts": 1.62,573 "n": 24574 },575 "C_conventional_rag": {576 "clarity": 5.96,577 "argument_logic": 5.88,578 "contribution_positioning": 5.21,579 "methodological_rigor": 5.21,580 "fidelity": 5.25,581 "usefulness": 5.67,582 "n_invented_facts": 1.71,583 "n": 24584 },585 "D_profiles_retrieval": {586 "clarity": 5.92,587 "argument_logic": 5.75,588 "contribution_positioning": 5.33,589 "methodological_rigor": 5.17,590 "fidelity": 5.25,591 "usefulness": 5.75,592 "n_invented_facts": 1.88,593 "n": 24594 },595 "E_full_engine": {596 "clarity": 5.83,597 "argument_logic": 6.04,598 "contribution_positioning": 4.92,599 "methodological_rigor": 6.12,600 "fidelity": 5.71,601 "usefulness": 5.42,602 "n_invented_facts": 0.88,603 "n": 24604 },605 "E_no_factcheck": {606 "clarity": 6.0,607 "argument_logic": 6.06,608 "contribution_positioning": 4.94,609 "methodological_rigor": 6.19,610 "fidelity": 5.75,611 "usefulness": 5.62,612 "n_invented_facts": 0.88,613 "n": 16614 },615 "E_no_profile": {616 "clarity": 5.94,617 "argument_logic": 6.12,618 "contribution_positioning": 5.12,619 "methodological_rigor": 6.38,620 "fidelity": 5.69,621 "usefulness": 5.69,622 "n_invented_facts": 1.0,623 "n": 16624 },625 "E_no_referee": {626 "clarity": 6.0,627 "argument_logic": 5.62,628 "contribution_positioning": 4.88,629 "methodological_rigor": 4.88,630 "fidelity": 5.12,631 "usefulness": 5.31,632 "n_invented_facts": 1.81,633 "n": 16634 },635 "E_no_retrieval": {636 "clarity": 5.88,637 "argument_logic": 5.94,638 "contribution_positioning": 5.19,639 "methodological_rigor": 6.19,640 "fidelity": 5.62,641 "usefulness": 5.56,642 "n_invented_facts": 1.5,643 "n": 16644 }645 },646 "task_means": {647 "('draft_introduction', 'A_baseline')": {648 "clarity": 6.25,649 "argument_logic": 6.25,650 "contribution_positioning": 5.5,651 "methodological_rigor": 5.25,652 "fidelity": 5.5,653 "usefulness": 6.0654 },655 "('draft_introduction', 'B_strong_prompt')": {656 "clarity": 6.25,657 "argument_logic": 6.25,658 "contribution_positioning": 6.25,659 "methodological_rigor": 5.25,660 "fidelity": 5.75,661 "usefulness": 6.0662 },663 "('draft_introduction', 'C_conventional_rag')": {664 "clarity": 6.0,665 "argument_logic": 6.0,666 "contribution_positioning": 5.25,667 "methodological_rigor": 5.5,668 "fidelity": 5.25,669 "usefulness": 6.0670 },671 "('draft_introduction', 'D_profiles_retrieval')": {672 "clarity": 5.75,673 "argument_logic": 6.25,674 "contribution_positioning": 5.5,675 "methodological_rigor": 5.0,676 "fidelity": 4.75,677 "usefulness": 5.75678 },679 "('draft_introduction', 'E_full_engine')": {680 "clarity": 6.0,681 "argument_logic": 6.25,682 "contribution_positioning": 4.25,683 "methodological_rigor": 6.0,684 "fidelity": 6.0,685 "usefulness": 5.0686 },687 "('draft_introduction', 'E_no_factcheck')": {688 "clarity": 6.0,689 "argument_logic": 6.5,690 "contribution_positioning": 4.0,691 "methodological_rigor": 6.5,692 "fidelity": 6.0,693 "usefulness": 5.5694 },695 "('draft_introduction', 'E_no_profile')": {696 "clarity": 6.0,697 "argument_logic": 5.5,698 "contribution_positioning": 4.0,699 "methodological_rigor": 6.0,700 "fidelity": 6.0,701 "usefulness": 5.0702 },703 "('draft_introduction', 'E_no_referee')": {704 "clarity": 6.0,705 "argument_logic": 6.0,706 "contribution_positioning": 5.0,707 "methodological_rigor": 5.0,708 "fidelity": 4.0,709 "usefulness": 5.0710 },711 "('draft_introduction', 'E_no_retrieval')": {712 "clarity": 5.5,713 "argument_logic": 5.5,714 "contribution_positioning": 4.0,715 "methodological_rigor": 5.5,716 "fidelity": 6.0,717 "usefulness": 5.0718 },719 "('explain_specification', 'A_baseline')": {720 "clarity": 6.0,721 "argument_logic": 6.0,722 "contribution_positioning": 5.0,723 "methodological_rigor": 5.0,724 "fidelity": 5.5,725 "usefulness": 6.0726 },727 "('explain_specification', 'B_strong_prompt')": {728 "clarity": 6.0,729 "argument_logic": 6.0,730 "contribution_positioning": 5.5,731 "methodological_rigor": 5.5,732 "fidelity": 5.0,733 "usefulness": 6.0734 },735 "('explain_specification', 'C_conventional_rag')": {736 "clarity": 6.0,737 "argument_logic": 6.0,738 "contribution_positioning": 5.0,739 "methodological_rigor": 6.0,740 "fidelity": 5.0,741 "usefulness": 6.0742 },743 "('explain_specification', 'D_profiles_retrieval')": {744 "clarity": 6.0,745 "argument_logic": 6.0,746 "contribution_positioning": 5.0,747 "methodological_rigor": 6.0,748 "fidelity": 5.5,749 "usefulness": 6.5750 },751 "('explain_specification', 'E_full_engine')": {752 "clarity": 5.5,753 "argument_logic": 6.0,754 "contribution_positioning": 5.0,755 "methodological_rigor": 6.0,756 "fidelity": 5.0,757 "usefulness": 5.0758 },759 "('explain_specification', 'E_no_factcheck')": {760 "clarity": 5.5,761 "argument_logic": 6.0,762 "contribution_positioning": 5.0,763 "methodological_rigor": 6.0,764 "fidelity": 5.5,765 "usefulness": 5.0766 },767 "('explain_specification', 'E_no_profile')": {768 "clarity": 5.5,769 "argument_logic": 6.0,770 "contribution_positioning": 5.0,771 "methodological_rigor": 6.0,772 "fidelity": 5.5,773 "usefulness": 5.0774 },775 "('explain_specification', 'E_no_referee')": {776 "clarity": 6.0,777 "argument_logic": 6.0,778 "contribution_positioning": 5.0,779 "methodological_rigor": 5.5,780 "fidelity": 5.0,781 "usefulness": 5.5782 },783 "('explain_specification', 'E_no_retrieval')": {784 "clarity": 5.5,785 "argument_logic": 6.0,786 "contribution_positioning": 5.0,787 "methodological_rigor": 6.5,788 "fidelity": 5.5,789 "usefulness": 5.5790 },791 "('flag_causal_claims', 'A_baseline')": {792 "clarity": 3.5,793 "argument_logic": 3.5,794 "contribution_positioning": 3.0,795 "methodological_rigor": 3.5,796 "fidelity": 3.5,797 "usefulness": 3.5798 },799 "('flag_causal_claims', 'B_strong_prompt')": {800 "clarity": 6.0,801 "argument_logic": 6.0,802 "contribution_positioning": 5.0,803 "methodological_rigor": 5.5,804 "fidelity": 5.5,805 "usefulness": 5.0806 },807 "('flag_causal_claims', 'C_conventional_rag')": {808 "clarity": 6.0,809 "argument_logic": 6.0,810 "contribution_positioning": 5.0,811 "methodological_rigor": 5.5,812 "fidelity": 6.0,813 "usefulness": 5.0814 },815 "('flag_causal_claims', 'D_profiles_retrieval')": {816 "clarity": 6.0,817 "argument_logic": 6.0,818 "contribution_positioning": 5.0,819 "methodological_rigor": 6.0,820 "fidelity": 6.0,821 "usefulness": 5.5822 },823 "('flag_causal_claims', 'E_full_engine')": {824 "clarity": 5.5,825 "argument_logic": 6.0,826 "contribution_positioning": 5.0,827 "methodological_rigor": 6.5,828 "fidelity": 6.0,829 "usefulness": 4.5830 },831 "('flag_causal_claims', 'E_no_factcheck')": {832 "clarity": 6.0,833 "argument_logic": 6.0,834 "contribution_positioning": 5.0,835 "methodological_rigor": 7.0,836 "fidelity": 5.5,837 "usefulness": 5.0838 },839 "('flag_causal_claims', 'E_no_profile')": {840 "clarity": 6.0,841 "argument_logic": 6.0,842 "contribution_positioning": 5.0,843 "methodological_rigor": 7.0,844 "fidelity": 6.0,845 "usefulness": 5.5846 },847 "('flag_causal_claims', 'E_no_referee')": {848 "clarity": 6.0,849 "argument_logic": 5.5,850 "contribution_positioning": 4.5,851 "methodological_rigor": 5.5,852 "fidelity": 6.0,853 "usefulness": 4.5854 },855 "('flag_causal_claims', 'E_no_retrieval')": {856 "clarity": 6.0,857 "argument_logic": 6.0,858 "contribution_positioning": 5.5,859 "methodological_rigor": 6.5,860 "fidelity": 5.5,861 "usefulness": 5.5862 },863 "('improve_abstract', 'A_baseline')": {864 "clarity": 6.0,865 "argument_logic": 5.75,866 "contribution_positioning": 5.5,867 "methodological_rigor": 4.0,868 "fidelity": 6.0,869 "usefulness": 6.0870 },871 "('improve_abstract', 'B_strong_prompt')": {872 "clarity": 6.25,873 "argument_logic": 5.25,874 "contribution_positioning": 5.0,875 "methodological_rigor": 4.0,876 "fidelity": 6.0,877 "usefulness": 5.75878 },879 "('improve_abstract', 'C_conventional_rag')": {880 "clarity": 6.0,881 "argument_logic": 6.0,882 "contribution_positioning": 5.25,883 "methodological_rigor": 4.5,884 "fidelity": 6.25,885 "usefulness": 5.75886 },887 "('improve_abstract', 'D_profiles_retrieval')": {888 "clarity": 6.0,889 "argument_logic": 5.5,890 "contribution_positioning": 5.75,891 "methodological_rigor": 4.25,892 "fidelity": 6.0,893 "usefulness": 6.0894 },895 "('improve_abstract', 'E_full_engine')": {896 "clarity": 6.0,897 "argument_logic": 6.0,898 "contribution_positioning": 5.25,899 "methodological_rigor": 5.5,900 "fidelity": 6.0,901 "usefulness": 6.0902 },903 "('improve_abstract', 'E_no_factcheck')": {904 "clarity": 6.0,905 "argument_logic": 6.0,906 "contribution_positioning": 5.5,907 "methodological_rigor": 5.5,908 "fidelity": 5.5,909 "usefulness": 6.0910 },911 "('improve_abstract', 'E_no_profile')": {912 "clarity": 6.0,913 "argument_logic": 6.0,914 "contribution_positioning": 5.5,915 "methodological_rigor": 6.0,916 "fidelity": 6.0,917 "usefulness": 6.0918 },919 "('improve_abstract', 'E_no_referee')": {920 "clarity": 6.0,921 "argument_logic": 5.5,922 "contribution_positioning": 5.0,923 "methodological_rigor": 4.0,924 "fidelity": 6.0,925 "usefulness": 5.5926 },927 "('improve_abstract', 'E_no_retrieval')": {928 "clarity": 6.0,929 "argument_logic": 6.0,930 "contribution_positioning": 5.5,931 "methodological_rigor": 6.0,932 "fidelity": 6.0,933 "usefulness": 6.0934 },935 "('improve_table', 'A_baseline')": {936 "clarity": 5.5,937 "argument_logic": 5.0,938 "contribution_positioning": 4.0,939 "methodological_rigor": 4.0,940 "fidelity": 5.0,941 "usefulness": 5.5942 },943 "('improve_table', 'B_strong_prompt')": {944 "clarity": 6.0,945 "argument_logic": 5.0,946 "contribution_positioning": 4.0,947 "methodological_rigor": 4.0,948 "fidelity": 5.0,949 "usefulness": 6.0950 },951 "('improve_table', 'C_conventional_rag')": {952 "clarity": 6.0,953 "argument_logic": 5.0,954 "contribution_positioning": 4.0,955 "methodological_rigor": 4.0,956 "fidelity": 4.5,957 "usefulness": 6.0958 },959 "('improve_table', 'D_profiles_retrieval')": {960 "clarity": 6.0,961 "argument_logic": 5.0,962 "contribution_positioning": 4.0,963 "methodological_rigor": 4.5,964 "fidelity": 5.5,965 "usefulness": 6.0966 },967 "('improve_table', 'E_full_engine')": {968 "clarity": 5.5,969 "argument_logic": 6.0,970 "contribution_positioning": 4.5,971 "methodological_rigor": 6.0,972 "fidelity": 5.5,973 "usefulness": 5.5974 },975 "('improve_table', 'E_no_factcheck')": {976 "clarity": 6.0,977 "argument_logic": 5.5,978 "contribution_positioning": 4.0,979 "methodological_rigor": 5.5,980 "fidelity": 5.5,981 "usefulness": 6.0982 },983 "('improve_table', 'E_no_profile')": {984 "clarity": 6.0,985 "argument_logic": 6.0,986 "contribution_positioning": 5.0,987 "methodological_rigor": 5.5,988 "fidelity": 5.5,989 "usefulness": 6.0990 },991 "('improve_table', 'E_no_referee')": {992 "clarity": 6.0,993 "argument_logic": 5.0,994 "contribution_positioning": 4.0,995 "methodological_rigor": 4.5,996 "fidelity": 5.5,997 "usefulness": 6.0998 },999 "('improve_table', 'E_no_retrieval')": {1000 "clarity": 6.0,1001 "argument_logic": 6.0,1002 "contribution_positioning": 5.0,1003 "methodological_rigor": 6.5,1004 "fidelity": 5.0,1005 "usefulness": 6.01006 },1007 "('interpret_results', 'A_baseline')": {1008 "clarity": 6.0,1009 "argument_logic": 5.5,1010 "contribution_positioning": 5.0,1011 "methodological_rigor": 4.75,1012 "fidelity": 5.0,1013 "usefulness": 5.51014 },1015 "('interpret_results', 'B_strong_prompt')": {1016 "clarity": 5.75,1017 "argument_logic": 5.5,1018 "contribution_positioning": 5.0,1019 "methodological_rigor": 4.75,1020 "fidelity": 4.5,1021 "usefulness": 5.251022 },1023 "('interpret_results', 'C_conventional_rag')": {1024 "clarity": 6.0,1025 "argument_logic": 5.75,1026 "contribution_positioning": 5.0,1027 "methodological_rigor": 4.75,1028 "fidelity": 4.25,1029 "usefulness": 5.51030 },1031 "('interpret_results', 'D_profiles_retrieval')": {1032 "clarity": 6.0,1033 "argument_logic": 5.5,1034 "contribution_positioning": 5.25,1035 "methodological_rigor": 5.0,1036 "fidelity": 4.25,1037 "usefulness": 5.51038 },1039 "('interpret_results', 'E_full_engine')": {1040 "clarity": 5.75,1041 "argument_logic": 6.0,1042 "contribution_positioning": 4.75,1043 "methodological_rigor": 6.75,1044 "fidelity": 5.5,1045 "usefulness": 5.51046 },1047 "('interpret_results', 'E_no_factcheck')": {1048 "clarity": 6.0,1049 "argument_logic": 6.0,1050 "contribution_positioning": 5.0,1051 "methodological_rigor": 6.5,1052 "fidelity": 6.0,1053 "usefulness": 6.01054 },1055 "('interpret_results', 'E_no_profile')": {1056 "clarity": 6.0,1057 "argument_logic": 6.5,1058 "contribution_positioning": 5.0,1059 "methodological_rigor": 7.0,1060 "fidelity": 6.0,1061 "usefulness": 6.01062 },1063 "('interpret_results', 'E_no_referee')": {1064 "clarity": 6.0,1065 "argument_logic": 5.0,1066 "contribution_positioning": 5.0,1067 "methodological_rigor": 4.0,1068 "fidelity": 4.0,1069 "usefulness": 5.51070 },1071 "('interpret_results', 'E_no_retrieval')": {1072 "clarity": 6.0,1073 "argument_logic": 6.0,1074 "contribution_positioning": 5.0,1075 "methodological_rigor": 6.5,1076 "fidelity": 5.5,1077 "usefulness": 5.51078 },1079 "('referee_report', 'A_baseline')": {1080 "clarity": 6.0,1081 "argument_logic": 6.0,1082 "contribution_positioning": 5.0,1083 "methodological_rigor": 6.25,1084 "fidelity": 6.0,1085 "usefulness": 5.751086 },1087 "('referee_report', 'B_strong_prompt')": {1088 "clarity": 6.0,1089 "argument_logic": 6.0,1090 "contribution_positioning": 5.75,1091 "methodological_rigor": 6.75,1092 "fidelity": 5.75,1093 "usefulness": 6.01094 },1095 "('referee_report', 'C_conventional_rag')": {1096 "clarity": 6.0,1097 "argument_logic": 6.25,1098 "contribution_positioning": 5.75,1099 "methodological_rigor": 6.5,1100 "fidelity": 5.75,1101 "usefulness": 5.751102 },1103 "('referee_report', 'D_profiles_retrieval')": {1104 "clarity": 6.0,1105 "argument_logic": 6.0,1106 "contribution_positioning": 5.5,1107 "methodological_rigor": 6.25,1108 "fidelity": 5.75,1109 "usefulness": 5.751110 },1111 "('referee_report', 'E_full_engine')": {1112 "clarity": 6.0,1113 "argument_logic": 5.75,1114 "contribution_positioning": 5.0,1115 "methodological_rigor": 6.0,1116 "fidelity": 6.0,1117 "usefulness": 5.51118 },1119 "('referee_report', 'E_no_factcheck')": {1120 "clarity": 6.0,1121 "argument_logic": 6.0,1122 "contribution_positioning": 5.0,1123 "methodological_rigor": 6.0,1124 "fidelity": 6.0,1125 "usefulness": 5.51126 },1127 "('referee_report', 'E_no_profile')": {1128 "clarity": 6.0,1129 "argument_logic": 6.0,1130 "contribution_positioning": 5.0,1131 "methodological_rigor": 6.5,1132 "fidelity": 5.5,1133 "usefulness": 6.01134 },1135 "('referee_report', 'E_no_referee')": {1136 "clarity": 6.0,1137 "argument_logic": 6.5,1138 "contribution_positioning": 5.5,1139 "methodological_rigor": 6.5,1140 "fidelity": 5.5,1141 "usefulness": 5.51142 },1143 "('referee_report', 'E_no_retrieval')": {1144 "clarity": 6.0,1145 "argument_logic": 6.0,1146 "contribution_positioning": 5.5,1147 "methodological_rigor": 6.0,1148 "fidelity": 6.5,1149 "usefulness": 5.51150 },1151 "('rewrite_literature', 'A_baseline')": {1152 "clarity": 6.0,1153 "argument_logic": 5.5,1154 "contribution_positioning": 6.0,1155 "methodological_rigor": 4.5,1156 "fidelity": 5.0,1157 "usefulness": 5.51158 },1159 "('rewrite_literature', 'B_strong_prompt')": {1160 "clarity": 6.0,1161 "argument_logic": 5.5,1162 "contribution_positioning": 5.5,1163 "methodological_rigor": 4.0,1164 "fidelity": 4.5,1165 "usefulness": 5.51166 },1167 "('rewrite_literature', 'C_conventional_rag')": {1168 "clarity": 5.5,1169 "argument_logic": 5.5,1170 "contribution_positioning": 6.0,1171 "methodological_rigor": 4.5,1172 "fidelity": 4.5,1173 "usefulness": 5.01174 },1175 "('rewrite_literature', 'D_profiles_retrieval')": {1176 "clarity": 5.5,1177 "argument_logic": 5.5,1178 "contribution_positioning": 6.0,1179 "methodological_rigor": 4.5,1180 "fidelity": 4.5,1181 "usefulness": 5.01182 },1183 "('rewrite_literature', 'E_full_engine')": {1184 "clarity": 6.0,1185 "argument_logic": 6.5,1186 "contribution_positioning": 6.0,1187 "methodological_rigor": 6.5,1188 "fidelity": 5.0,1189 "usefulness": 6.01190 },1191 "('rewrite_literature', 'E_no_factcheck')": {1192 "clarity": 6.5,1193 "argument_logic": 6.5,1194 "contribution_positioning": 6.0,1195 "methodological_rigor": 6.5,1196 "fidelity": 6.0,1197 "usefulness": 6.01198 },1199 "('rewrite_literature', 'E_no_profile')": {1200 "clarity": 6.0,1201 "argument_logic": 7.0,1202 "contribution_positioning": 6.5,1203 "methodological_rigor": 7.0,1204 "fidelity": 5.0,1205 "usefulness": 6.01206 },1207 "('rewrite_literature', 'E_no_referee')": {1208 "clarity": 6.0,1209 "argument_logic": 5.5,1210 "contribution_positioning": 5.0,1211 "methodological_rigor": 4.0,1212 "fidelity": 5.0,1213 "usefulness": 5.01214 },1215 "('rewrite_literature', 'E_no_retrieval')": {1216 "clarity": 6.0,1217 "argument_logic": 6.0,1218 "contribution_positioning": 6.0,1219 "methodological_rigor": 6.0,1220 "fidelity": 5.0,1221 "usefulness": 5.51222 }1223 },1224 "out_of_domain_means": {1225 "A_baseline": {1226 "clarity": 6.0,1227 "argument_logic": 6.0,1228 "contribution_positioning": 5.67,1229 "methodological_rigor": 5.67,1230 "fidelity": 5.0,1231 "usefulness": 5.331232 },1233 "B_strong_prompt": {1234 "clarity": 6.0,1235 "argument_logic": 6.0,1236 "contribution_positioning": 6.0,1237 "methodological_rigor": 5.33,1238 "fidelity": 4.67,1239 "usefulness": 5.331240 },1241 "C_conventional_rag": {1242 "clarity": 6.0,1243 "argument_logic": 5.67,1244 "contribution_positioning": 5.67,1245 "methodological_rigor": 5.0,1246 "fidelity": 4.33,1247 "usefulness": 5.331248 },1249 "D_profiles_retrieval": {1250 "clarity": 5.33,1251 "argument_logic": 5.67,1252 "contribution_positioning": 5.33,1253 "methodological_rigor": 5.67,1254 "fidelity": 4.67,1255 "usefulness": 4.671256 },1257 "E_full_engine": {1258 "clarity": 5.67,1259 "argument_logic": 6.0,1260 "contribution_positioning": 5.33,1261 "methodological_rigor": 5.67,1262 "fidelity": 4.33,1263 "usefulness": 5.01264 }1265 },1266 "judge_consistency": {1267 "n_pairs": 50,1268 "clarity": {1269 "mean_abs_diff": 0.26,1270 "spearman": 0.4111271 },1272 "argument_logic": {1273 "mean_abs_diff": 0.3,1274 "spearman": 0.5821275 },1276 "contribution_positioning": {1277 "mean_abs_diff": 0.32,1278 "spearman": 0.841279 },1280 "methodological_rigor": {1281 "mean_abs_diff": 0.34,1282 "spearman": 0.8041283 },1284 "fidelity": {1285 "mean_abs_diff": 0.3,1286 "spearman": 0.7471287 },1288 "usefulness": {1289 "mean_abs_diff": 0.4,1290 "spearman": 0.5311291 }1292 },1293 "secondary_means": {1294 "A_baseline": {1295 "preserved_rate": 0.053,1296 "new_numbers": 1.0,1297 "n_unverifiable_citations": 0.083,1298 "n_unsupported_claims": 1.083,1299 "max_overlap_share": 0.073,1300 "todo_placeholders": 2.542,1301 "n_words": 959.333,1302 "cost_usd": 0.138,1303 "latency_s": 45.879,1304 "n": 241305 },1306 "B_strong_prompt": {1307 "preserved_rate": 0.057,1308 "new_numbers": 1.042,1309 "n_unverifiable_citations": 0.083,1310 "n_unsupported_claims": 1.042,1311 "max_overlap_share": 0.065,1312 "todo_placeholders": 2.958,1313 "n_words": 1027.417,1314 "cost_usd": 0.139,1315 "latency_s": 46.083,1316 "n": 241317 },1318 "C_conventional_rag": {1319 "preserved_rate": 0.053,1320 "new_numbers": 1.25,1321 "n_unverifiable_citations": 0.083,1322 "n_unsupported_claims": 1.625,1323 "max_overlap_share": 0.072,1324 "todo_placeholders": 2.167,1325 "n_words": 1059.875,1326 "cost_usd": 0.146,1327 "latency_s": 47.446,1328 "n": 241329 },1330 "D_profiles_retrieval": {1331 "preserved_rate": 0.055,1332 "new_numbers": 1.167,1333 "n_unverifiable_citations": 0.083,1334 "n_unsupported_claims": 1.208,1335 "max_overlap_share": 0.068,1336 "todo_placeholders": 2.667,1337 "n_words": 1051.875,1338 "cost_usd": 0.154,1339 "latency_s": 49.475,1340 "n": 241341 },1342 "E_full_engine": {1343 "preserved_rate": 0.055,1344 "new_numbers": 0.833,1345 "n_unverifiable_citations": 0.0,1346 "n_unsupported_claims": 0.625,1347 "max_overlap_share": 0.059,1348 "todo_placeholders": 3.125,1349 "n_words": 994.667,1350 "cost_usd": 0.316,1351 "latency_s": 98.033,1352 "n": 241353 },1354 "E_no_factcheck": {1355 "preserved_rate": 0.048,1356 "new_numbers": 0.938,1357 "n_unverifiable_citations": 0.062,1358 "n_unsupported_claims": 0.75,1359 "max_overlap_share": 0.057,1360 "todo_placeholders": 2.562,1361 "n_words": 1049.938,1362 "cost_usd": 0.312,1363 "latency_s": 95.925,1364 "n": 161365 },1366 "E_no_profile": {1367 "preserved_rate": 0.054,1368 "new_numbers": 0.688,1369 "n_unverifiable_citations": 0.0,1370 "n_unsupported_claims": 0.312,1371 "max_overlap_share": 0.06,1372 "todo_placeholders": 2.75,1373 "n_words": 996.188,1374 "cost_usd": 0.311,1375 "latency_s": 97.706,1376 "n": 161377 },1378 "E_no_referee": {1379 "preserved_rate": 0.044,1380 "new_numbers": 0.875,1381 "n_unverifiable_citations": 0.125,1382 "n_unsupported_claims": 1.062,1383 "max_overlap_share": 0.076,1384 "todo_placeholders": 2.25,1385 "n_words": 939.188,1386 "cost_usd": 0.155,1387 "latency_s": 48.856,1388 "n": 161389 },1390 "E_no_retrieval": {1391 "preserved_rate": 0.042,1392 "new_numbers": 0.625,1393 "n_unverifiable_citations": 0.125,1394 "n_unsupported_claims": 0.312,1395 "max_overlap_share": 0.055,1396 "todo_placeholders": 5.062,1397 "n_words": 1074.375,1398 "cost_usd": 0.308,1399 "latency_s": 94.531,1400 "n": 161401 }1402 },1403 "H4": {1404 "E_vs_A": {1405 "n_unverifiable_citations": {1406 "n": 24,1407 "mean_diff": -0.083,1408 "ci95": [1409 -0.25,1410 0.01411 ],1412 "wilcoxon_p": 0.15729920705028505,1413 "cliffs_delta": -0.0831414 },1415 "n_unsupported_claims": {1416 "n": 24,1417 "mean_diff": -0.458,1418 "ci95": [1419 -0.906,1420 0.251421 ],1422 "wilcoxon_p": 0.1005626399172922,1423 "cliffs_delta": -0.1631424 },1425 "new_numbers": {1426 "n": 24,1427 "mean_diff": -0.167,1428 "ci95": [1429 -0.375,1430 0.1251431 ],1432 "wilcoxon_p": 0.15729920705028516,1433 "cliffs_delta": -0.1011434 }1435 },1436 "E_vs_B": {1437 "n_unverifiable_citations": {1438 "n": 24,1439 "mean_diff": -0.083,1440 "ci95": [1441 -0.156,1442 0.01443 ],1444 "wilcoxon_p": 0.15729920705028505,1445 "cliffs_delta": -0.0831446 },1447 "n_unsupported_claims": {1448 "n": 24,1449 "mean_diff": -0.417,1450 "ci95": [1451 -0.938,1452 0.1881453 ],1454 "wilcoxon_p": 0.17298970944016634,1455 "cliffs_delta": -0.1811456 },1457 "new_numbers": {1458 "n": 24,1459 "mean_diff": -0.208,1460 "ci95": [1461 -0.406,1462 0.0621463 ],1464 "wilcoxon_p": 0.09558070454562939,1465 "cliffs_delta": -0.1411466 }1467 },1468 "E_vs_C": {1469 "n_unverifiable_citations": {1470 "n": 24,1471 "mean_diff": -0.083,1472 "ci95": [1473 -0.312,1474 0.01475 ],1476 "wilcoxon_p": 0.15729920705028505,1477 "cliffs_delta": -0.0831478 },1479 "n_unsupported_claims": {1480 "n": 24,1481 "mean_diff": -1.0,1482 "ci95": [1483 -1.594,1484 -0.1561485 ],1486 "wilcoxon_p": 0.008554328993751146,1487 "cliffs_delta": -0.3661488 },1489 "new_numbers": {1490 "n": 24,1491 "mean_diff": -0.417,1492 "ci95": [1493 -1.0,1494 0.1251495 ],1496 "wilcoxon_p": 0.31731050786291415,1497 "cliffs_delta": -0.0821498 }1499 }1500 },1501 "H5_overlap": {1502 "preregistered_metric": {1503 "max_overlap_share_overall": 0.4044,1504 "share_outputs_below_0.02": 0.418478260869565241505 },1506 "excluding_source_document": {1507 "max_other_share": 0.0069,1508 "share_outputs_below_0.02": 1.0,1509 "mean_other_share": 0.00079239130434782611510 },1511 "self_overlap_by_task": {1512 "draft_introduction": 0.028,1513 "explain_specification": 0.019,1514 "flag_causal_claims": 0.073,1515 "improve_abstract": 0.253,1516 "improve_table": 0.02,1517 "interpret_results": 0.018,1518 "referee_report": 0.008,1519 "rewrite_literature": 0.0821520 },1521 "self_overlap_by_condition": {1522 "A_baseline": 0.073,1523 "B_strong_prompt": 0.065,1524 "C_conventional_rag": 0.072,1525 "D_profiles_retrieval": 0.068,1526 "E_full_engine": 0.059,1527 "E_no_factcheck": 0.057,1528 "E_no_profile": 0.06,1529 "E_no_referee": 0.076,1530 "E_no_retrieval": 0.0551531 }1532 }1533}