Repository Public Research preview
1{2 "created_at": "2026-11-10T17:18:12Z",3 "git": {4 "commit": "883ce93e2f627cf3d42040460687c511ef436337",5 "branch": "main",6 "dirty": true7 },8 "libraries": {9 "python": "3.14.4",10 "torch": "2.12.0",11 "numpy": "2.5.4",12 "anthropic": "2.3.0",13 "transformers": "5.8.1",14 "sentencepiece": "0.2.2",15 "lxml": "6.1.3",16 "pyarrow": "24.0.0",17 "sacrebleu": "2.6.0",18 "datasketch": "2.0.0",19 "rapidfuzz": "3.14.6"20 },21 "hardware": {22 "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",23 "machine": "arm64",24 "cpu_count": 18,25 "chip": "Apple M5 Max",26 "ram_gb": 48.0,27 "torch_mps": true,28 "torch_cuda": false29 },30 "n_pairs": 812,31 "systems": {32 "B0_identity": {33 "overall": {34 "n": 812,35 "chrf": {36 "mean": 69.89504762796885,37 "ci_low": 68.94469598805215,38 "ci_high": 71.0690255654406,39 "n_items": 812,40 "n_groups": 5441 },42 "edit_sim": {43 "mean": 0.6599015087792276,44 "ci_low": 0.6474891028902454,45 "ci_high": 0.6720575753534699,46 "n_items": 812,47 "n_groups": 5448 },49 "rougeL": {50 "mean": 0.670493386449416,51 "ci_low": 0.658487990094312,52 "ci_high": 0.684408333535875,53 "n_items": 812,54 "n_groups": 5455 },56 "exact_match": {57 "mean": 0.0,58 "ci_low": 0.0,59 "ci_high": 0.0,60 "n_items": 812,61 "n_groups": 5462 },63 "number_recall": {64 "mean": 1.0,65 "ci_low": 1.0,66 "ci_high": 1.0,67 "n_items": 812,68 "n_groups": 5469 },70 "placeholders_exact": {71 "mean": 1.0,72 "ci_low": 1.0,73 "ci_high": 1.0,74 "n_items": 812,75 "n_groups": 5476 }77 },78 "by_variant": {79 "A": {80 "n": 133,81 "chrf": {82 "mean": 67.43425994431013,83 "ci_low": 66.47910218568393,84 "ci_high": 68.541354861052,85 "n_items": 133,86 "n_groups": 5087 },88 "edit_sim": {89 "mean": 0.614689128259703,90 "ci_low": 0.6001454429426467,91 "ci_high": 0.627298508762089,92 "n_items": 133,93 "n_groups": 5094 },95 "rougeL": {96 "mean": 0.6251200906716861,97 "ci_low": 0.6120108952634302,98 "ci_high": 0.638850465763199,99 "n_items": 133,100 "n_groups": 50101 },102 "exact_match": {103 "mean": 0.0,104 "ci_low": 0.0,105 "ci_high": 0.0,106 "n_items": 133,107 "n_groups": 50108 },109 "number_recall": {110 "mean": 1.0,111 "ci_low": 1.0,112 "ci_high": 1.0,113 "n_items": 133,114 "n_groups": 50115 },116 "placeholders_exact": {117 "mean": 1.0,118 "ci_low": 1.0,119 "ci_high": 1.0,120 "n_items": 133,121 "n_groups": 50122 }123 },124 "B": {125 "n": 136,126 "chrf": {127 "mean": 78.302958308022,128 "ci_low": 76.95172601691728,129 "ci_high": 79.69392827451422,130 "n_items": 136,131 "n_groups": 52132 },133 "edit_sim": {134 "mean": 0.7676751038593954,135 "ci_low": 0.751414264083352,136 "ci_high": 0.7815566907712,137 "n_items": 136,138 "n_groups": 52139 },140 "rougeL": {141 "mean": 0.7778418860516894,142 "ci_low": 0.7625217048745268,143 "ci_high": 0.7939729675709926,144 "n_items": 136,145 "n_groups": 52146 },147 "exact_match": {148 "mean": 0.0,149 "ci_low": 0.0,150 "ci_high": 0.0,151 "n_items": 136,152 "n_groups": 52153 },154 "number_recall": {155 "mean": 1.0,156 "ci_low": 1.0,157 "ci_high": 1.0,158 "n_items": 136,159 "n_groups": 52160 },161 "placeholders_exact": {162 "mean": 1.0,163 "ci_low": 1.0,164 "ci_high": 1.0,165 "n_items": 136,166 "n_groups": 52167 }168 },169 "C": {170 "n": 135,171 "chrf": {172 "mean": 73.3854104705856,173 "ci_low": 72.01395154280281,174 "ci_high": 75.02785716404159,175 "n_items": 135,176 "n_groups": 53177 },178 "edit_sim": {179 "mean": 0.7330342767003942,180 "ci_low": 0.7193806373980995,181 "ci_high": 0.7478218558153281,182 "n_items": 135,183 "n_groups": 53184 },185 "rougeL": {186 "mean": 0.7545894903650573,187 "ci_low": 0.739841157205696,188 "ci_high": 0.7719964716271877,189 "n_items": 135,190 "n_groups": 53191 },192 "exact_match": {193 "mean": 0.0,194 "ci_low": 0.0,195 "ci_high": 0.0,196 "n_items": 135,197 "n_groups": 53198 },199 "number_recall": {200 "mean": 1.0,201 "ci_low": 1.0,202 "ci_high": 1.0,203 "n_items": 135,204 "n_groups": 53205 },206 "placeholders_exact": {207 "mean": 1.0,208 "ci_low": 1.0,209 "ci_high": 1.0,210 "n_items": 135,211 "n_groups": 53212 }213 },214 "D": {215 "n": 136,216 "chrf": {217 "mean": 65.47915813968179,218 "ci_low": 64.2786431676907,219 "ci_high": 66.71958942463182,220 "n_items": 136,221 "n_groups": 51222 },223 "edit_sim": {224 "mean": 0.5920839058015158,225 "ci_low": 0.5768940705007222,226 "ci_high": 0.6078472395025183,227 "n_items": 136,228 "n_groups": 51229 },230 "rougeL": {231 "mean": 0.6005607520951276,232 "ci_low": 0.5863841558517404,233 "ci_high": 0.6158460693668083,234 "n_items": 136,235 "n_groups": 51236 },237 "exact_match": {238 "mean": 0.0,239 "ci_low": 0.0,240 "ci_high": 0.0,241 "n_items": 136,242 "n_groups": 51243 },244 "number_recall": {245 "mean": 1.0,246 "ci_low": 1.0,247 "ci_high": 1.0,248 "n_items": 136,249 "n_groups": 51250 },251 "placeholders_exact": {252 "mean": 1.0,253 "ci_low": 1.0,254 "ci_high": 1.0,255 "n_items": 136,256 "n_groups": 51257 }258 },259 "E": {260 "n": 272,261 "chrf": {262 "mean": 67.36994062831138,263 "ci_low": 66.35873519898063,264 "ci_high": 68.57249628615695,265 "n_items": 272,266 "n_groups": 52267 },268 "edit_sim": {269 "mean": 0.6257335235358646,270 "ci_low": 0.6097093074819373,271 "ci_high": 0.6426608883870822,272 "n_items": 272,273 "n_groups": 52274 },275 "rougeL": {276 "mean": 0.6322327858468446,277 "ci_low": 0.6181652548151976,278 "ci_high": 0.6483200570670029,279 "n_items": 272,280 "n_groups": 52281 },282 "exact_match": {283 "mean": 0.0,284 "ci_low": 0.0,285 "ci_high": 0.0,286 "n_items": 272,287 "n_groups": 52288 },289 "number_recall": {290 "mean": 1.0,291 "ci_low": 1.0,292 "ci_high": 1.0,293 "n_items": 272,294 "n_groups": 52295 },296 "placeholders_exact": {297 "mean": 1.0,298 "ci_low": 1.0,299 "ci_high": 1.0,300 "n_items": 272,301 "n_groups": 52302 }303 }304 },305 "by_discipline": {306 "biomedicine": {307 "n": 380,308 "chrf": {309 "mean": 70.39477977144168,310 "ci_low": 69.22442340287664,311 "ci_high": 71.5561915300661,312 "n_items": 380,313 "n_groups": 31314 },315 "edit_sim": {316 "mean": 0.6577017733583121,317 "ci_low": 0.6401545492164868,318 "ci_high": 0.6713756110164587,319 "n_items": 380,320 "n_groups": 31321 },322 "rougeL": {323 "mean": 0.6729874211260962,324 "ci_low": 0.657133719990892,325 "ci_high": 0.6848287103955965,326 "n_items": 380,327 "n_groups": 31328 },329 "exact_match": {330 "mean": 0.0,331 "ci_low": 0.0,332 "ci_high": 0.0,333 "n_items": 380,334 "n_groups": 31335 },336 "number_recall": {337 "mean": 1.0,338 "ci_low": 1.0,339 "ci_high": 1.0,340 "n_items": 380,341 "n_groups": 31342 },343 "placeholders_exact": {344 "mean": 1.0,345 "ci_low": 1.0,346 "ci_high": 1.0,347 "n_items": 380,348 "n_groups": 31349 }350 },351 "engineering": {352 "n": 12,353 "chrf": {354 "mean": 73.22145888212934,355 "ci_low": 72.92571178933999,356 "ci_high": 73.51720597491868,357 "n_items": 12,358 "n_groups": 2359 },360 "edit_sim": {361 "mean": 0.7040423213898451,362 "ci_low": 0.700118191596903,363 "ci_high": 0.7079664511827871,364 "n_items": 12,365 "n_groups": 2366 },367 "rougeL": {368 "mean": 0.7036156017284799,369 "ci_low": 0.6818259849148215,370 "ci_high": 0.7254052185421385,371 "n_items": 12,372 "n_groups": 2373 },374 "exact_match": {375 "mean": 0.0,376 "ci_low": 0.0,377 "ci_high": 0.0,378 "n_items": 12,379 "n_groups": 2380 },381 "number_recall": {382 "mean": 1.0,383 "ci_low": 1.0,384 "ci_high": 1.0,385 "n_items": 12,386 "n_groups": 2387 },388 "placeholders_exact": {389 "mean": 1.0,390 "ci_low": 1.0,391 "ci_high": 1.0,392 "n_items": 12,393 "n_groups": 2394 }395 },396 "finance": {397 "n": 96,398 "chrf": {399 "mean": 68.02685046237909,400 "ci_low": 68.02685046237909,401 "ci_high": 68.02685046237909,402 "n_items": 96,403 "n_groups": 1404 },405 "edit_sim": {406 "mean": 0.6564396805059274,407 "ci_low": 0.6564396805059274,408 "ci_high": 0.6564396805059274,409 "n_items": 96,410 "n_groups": 1411 },412 "rougeL": {413 "mean": 0.6426837689762521,414 "ci_low": 0.6426837689762521,415 "ci_high": 0.6426837689762521,416 "n_items": 96,417 "n_groups": 1418 },419 "exact_match": {420 "mean": 0.0,421 "ci_low": 0.0,422 "ci_high": 0.0,423 "n_items": 96,424 "n_groups": 1425 },426 "number_recall": {427 "mean": 1.0,428 "ci_low": 1.0,429 "ci_high": 1.0,430 "n_items": 96,431 "n_groups": 1432 },433 "placeholders_exact": {434 "mean": 1.0,435 "ci_low": 1.0,436 "ci_high": 1.0,437 "n_items": 96,438 "n_groups": 1439 }440 },441 "life_sciences": {442 "n": 162,443 "chrf": {444 "mean": 71.61358582971289,445 "ci_low": 68.86204653965949,446 "ci_high": 74.1119172747608,447 "n_items": 162,448 "n_groups": 15449 },450 "edit_sim": {451 "mean": 0.6750021596788679,452 "ci_low": 0.6516168007989388,453 "ci_high": 0.6980281132991072,454 "n_items": 162,455 "n_groups": 15456 },457 "rougeL": {458 "mean": 0.695332441919705,459 "ci_low": 0.6687243934298359,460 "ci_high": 0.7212339474913836,461 "n_items": 162,462 "n_groups": 15463 },464 "exact_match": {465 "mean": 0.0,466 "ci_low": 0.0,467 "ci_high": 0.0,468 "n_items": 162,469 "n_groups": 15470 },471 "number_recall": {472 "mean": 1.0,473 "ci_low": 1.0,474 "ci_high": 1.0,475 "n_items": 162,476 "n_groups": 15477 },478 "placeholders_exact": {479 "mean": 1.0,480 "ci_low": 1.0,481 "ci_high": 1.0,482 "n_items": 162,483 "n_groups": 15484 }485 },486 "mathematics": {487 "n": 60,488 "chrf": {489 "mean": 68.47720706370636,490 "ci_low": 68.47720706370636,491 "ci_high": 68.47720706370636,492 "n_items": 60,493 "n_groups": 1494 },495 "edit_sim": {496 "mean": 0.6819062624885746,497 "ci_low": 0.6819062624885746,498 "ci_high": 0.6819062624885746,499 "n_items": 60,500 "n_groups": 1501 },502 "rougeL": {503 "mean": 0.664019840152223,504 "ci_low": 0.664019840152223,505 "ci_high": 0.664019840152223,506 "n_items": 60,507 "n_groups": 1508 },509 "exact_match": {510 "mean": 0.0,511 "ci_low": 0.0,512 "ci_high": 0.0,513 "n_items": 60,514 "n_groups": 1515 },516 "number_recall": {517 "mean": 1.0,518 "ci_low": 1.0,519 "ci_high": 1.0,520 "n_items": 60,521 "n_groups": 1522 },523 "placeholders_exact": {524 "mean": 1.0,525 "ci_low": 1.0,526 "ci_high": 1.0,527 "n_items": 60,528 "n_groups": 1529 }530 },531 "social_sciences": {532 "n": 30,533 "chrf": {534 "mean": 68.76932559874858,535 "ci_low": 56.462466981189145,536 "ci_high": 71.84604025313844,537 "n_items": 30,538 "n_groups": 2539 },540 "edit_sim": {541 "mean": 0.6890367251839173,542 "ci_low": 0.5823213981188687,543 "ci_high": 0.7157155569501795,544 "n_items": 30,545 "n_groups": 2546 },547 "rougeL": {548 "mean": 0.6792392726039741,549 "ci_low": 0.48512028470313967,550 "ci_high": 0.7277690195791827,551 "n_items": 30,552 "n_groups": 2553 },554 "exact_match": {555 "mean": 0.0,556 "ci_low": 0.0,557 "ci_high": 0.0,558 "n_items": 30,559 "n_groups": 2560 },561 "number_recall": {562 "mean": 1.0,563 "ci_low": 1.0,564 "ci_high": 1.0,565 "n_items": 30,566 "n_groups": 2567 },568 "placeholders_exact": {569 "mean": 1.0,570 "ci_low": 1.0,571 "ci_high": 1.0,572 "n_items": 30,573 "n_groups": 2574 }575 },576 "statistics": {577 "n": 72,578 "chrf": {579 "mean": 66.97797379986946,580 "ci_low": 64.55420417704453,581 "ci_high": 68.18985861128193,582 "n_items": 72,583 "n_groups": 2584 },585 "edit_sim": {586 "mean": 0.604316759646089,587 "ci_low": 0.5823774620494792,588 "ci_high": 0.615286408444394,589 "n_items": 72,590 "n_groups": 2591 },592 "rougeL": {593 "mean": 0.6347521742819794,594 "ci_low": 0.6298658757181202,595 "ci_high": 0.6371953235639091,596 "n_items": 72,597 "n_groups": 2598 },599 "exact_match": {600 "mean": 0.0,601 "ci_low": 0.0,602 "ci_high": 0.0,603 "n_items": 72,604 "n_groups": 2605 },606 "number_recall": {607 "mean": 1.0,608 "ci_low": 1.0,609 "ci_high": 1.0,610 "n_items": 72,611 "n_groups": 2612 },613 "placeholders_exact": {614 "mean": 1.0,615 "ci_low": 1.0,616 "ci_high": 1.0,617 "n_items": 72,618 "n_groups": 2619 }620 }621 },622 "by_era": {623 "1980-1999": {624 "n": 8,625 "chrf": {626 "mean": 72.81439832446404,627 "ci_low": 70.76269961800656,628 "ci_high": 76.23389616855984,629 "n_items": 8,630 "n_groups": 2631 },632 "edit_sim": {633 "mean": 0.7066743911673423,634 "ci_low": 0.6754591510514822,635 "ci_high": 0.7586997913604426,636 "n_items": 8,637 "n_groups": 2638 },639 "rougeL": {640 "mean": 0.7083590141113916,641 "ci_low": 0.6771684622905808,642 "ci_high": 0.7603432671460763,643 "n_items": 8,644 "n_groups": 2645 },646 "exact_match": {647 "mean": 0.0,648 "ci_low": 0.0,649 "ci_high": 0.0,650 "n_items": 8,651 "n_groups": 2652 },653 "number_recall": {654 "mean": 1.0,655 "ci_low": 1.0,656 "ci_high": 1.0,657 "n_items": 8,658 "n_groups": 2659 },660 "placeholders_exact": {661 "mean": 1.0,662 "ci_low": 1.0,663 "ci_high": 1.0,664 "n_items": 8,665 "n_groups": 2666 }667 },668 "2000-2009": {669 "n": 89,670 "chrf": {671 "mean": 68.56386383312753,672 "ci_low": 66.1428711916568,673 "ci_high": 72.5919305863491,674 "n_items": 89,675 "n_groups": 7676 },677 "edit_sim": {678 "mean": 0.6458796855380733,679 "ci_low": 0.6118167847532366,680 "ci_high": 0.6961706345570695,681 "n_items": 89,682 "n_groups": 7683 },684 "rougeL": {685 "mean": 0.6603264567254922,686 "ci_low": 0.6380317856534418,687 "ci_high": 0.7082963518224497,688 "n_items": 89,689 "n_groups": 7690 },691 "exact_match": {692 "mean": 0.0,693 "ci_low": 0.0,694 "ci_high": 0.0,695 "n_items": 89,696 "n_groups": 7697 },698 "number_recall": {699 "mean": 1.0,700 "ci_low": 1.0,701 "ci_high": 1.0,702 "n_items": 89,703 "n_groups": 7704 },705 "placeholders_exact": {706 "mean": 1.0,707 "ci_low": 1.0,708 "ci_high": 1.0,709 "n_items": 89,710 "n_groups": 7711 }712 },713 "2010-2017": {714 "n": 715,715 "chrf": {716 "mean": 70.02808336526805,717 "ci_low": 69.02513737502876,718 "ci_high": 71.27196365060998,719 "n_items": 715,720 "n_groups": 45721 },722 "edit_sim": {723 "mean": 0.6611235496314763,724 "ci_low": 0.6480911632230502,725 "ci_high": 0.6734080785929272,726 "n_items": 715,727 "n_groups": 45728 },729 "rougeL": {730 "mean": 0.6713352490006517,731 "ci_low": 0.6577275195515899,732 "ci_high": 0.6861741413830451,733 "n_items": 715,734 "n_groups": 45735 },736 "exact_match": {737 "mean": 0.0,738 "ci_low": 0.0,739 "ci_high": 0.0,740 "n_items": 715,741 "n_groups": 45742 },743 "number_recall": {744 "mean": 1.0,745 "ci_low": 1.0,746 "ci_high": 1.0,747 "n_items": 715,748 "n_groups": 45749 },750 "placeholders_exact": {751 "mean": 1.0,752 "ci_low": 1.0,753 "ci_high": 1.0,754 "n_items": 715,755 "n_groups": 45756 }757 }758 },759 "by_role": {760 "abstract": {761 "n": 83,762 "chrf": {763 "mean": 72.19355877759274,764 "ci_low": 70.60949518764066,765 "ci_high": 73.80469386714313,766 "n_items": 83,767 "n_groups": 15768 },769 "edit_sim": {770 "mean": 0.6833226919062769,771 "ci_low": 0.6604404247369527,772 "ci_high": 0.7040941939260852,773 "n_items": 83,774 "n_groups": 15775 },776 "rougeL": {777 "mean": 0.6942576928785549,778 "ci_low": 0.6712839314511271,779 "ci_high": 0.7176565884968416,780 "n_items": 83,781 "n_groups": 15782 },783 "exact_match": {784 "mean": 0.0,785 "ci_low": 0.0,786 "ci_high": 0.0,787 "n_items": 83,788 "n_groups": 15789 },790 "number_recall": {791 "mean": 1.0,792 "ci_low": 1.0,793 "ci_high": 1.0,794 "n_items": 83,795 "n_groups": 15796 },797 "placeholders_exact": {798 "mean": 1.0,799 "ci_low": 1.0,800 "ci_high": 1.0,801 "n_items": 83,802 "n_groups": 15803 }804 },805 "background": {806 "n": 58,807 "chrf": {808 "mean": 69.67596825778382,809 "ci_low": 66.53570339798188,810 "ci_high": 72.54772208514592,811 "n_items": 58,812 "n_groups": 7813 },814 "edit_sim": {815 "mean": 0.666077973061832,816 "ci_low": 0.6387417325044762,817 "ci_high": 0.6973349148410127,818 "n_items": 58,819 "n_groups": 7820 },821 "rougeL": {822 "mean": 0.675395907082031,823 "ci_low": 0.652304846121757,824 "ci_high": 0.7003761065149141,825 "n_items": 58,826 "n_groups": 7827 },828 "exact_match": {829 "mean": 0.0,830 "ci_low": 0.0,831 "ci_high": 0.0,832 "n_items": 58,833 "n_groups": 7834 },835 "number_recall": {836 "mean": 1.0,837 "ci_low": 1.0,838 "ci_high": 1.0,839 "n_items": 58,840 "n_groups": 7841 },842 "placeholders_exact": {843 "mean": 1.0,844 "ci_low": 1.0,845 "ci_high": 1.0,846 "n_items": 58,847 "n_groups": 7848 }849 },850 "conclusion": {851 "n": 47,852 "chrf": {853 "mean": 68.03482029181029,854 "ci_low": 65.75541896033377,855 "ci_high": 69.9156262229125,856 "n_items": 47,857 "n_groups": 8858 },859 "edit_sim": {860 "mean": 0.6602906342089576,861 "ci_low": 0.631003759285465,862 "ci_high": 0.6883191226569255,863 "n_items": 47,864 "n_groups": 8865 },866 "rougeL": {867 "mean": 0.6499366473915379,868 "ci_low": 0.6215529781047012,869 "ci_high": 0.6754831786097706,870 "n_items": 47,871 "n_groups": 8872 },873 "exact_match": {874 "mean": 0.0,875 "ci_low": 0.0,876 "ci_high": 0.0,877 "n_items": 47,878 "n_groups": 8879 },880 "number_recall": {881 "mean": 1.0,882 "ci_low": 1.0,883 "ci_high": 1.0,884 "n_items": 47,885 "n_groups": 8886 },887 "placeholders_exact": {888 "mean": 1.0,889 "ci_low": 1.0,890 "ci_high": 1.0,891 "n_items": 47,892 "n_groups": 8893 }894 },895 "data": {896 "n": 69,897 "chrf": {898 "mean": 71.86325658669003,899 "ci_low": 68.65452198237378,900 "ci_high": 75.56061341558298,901 "n_items": 69,902 "n_groups": 10903 },904 "edit_sim": {905 "mean": 0.6352349563947333,906 "ci_low": 0.585866649535811,907 "ci_high": 0.6896433658023071,908 "n_items": 69,909 "n_groups": 10910 },911 "rougeL": {912 "mean": 0.6743645071130052,913 "ci_low": 0.6340041836687129,914 "ci_high": 0.7244316231406165,915 "n_items": 69,916 "n_groups": 10917 },918 "exact_match": {919 "mean": 0.0,920 "ci_low": 0.0,921 "ci_high": 0.0,922 "n_items": 69,923 "n_groups": 10924 },925 "number_recall": {926 "mean": 1.0,927 "ci_low": 1.0,928 "ci_high": 1.0,929 "n_items": 69,930 "n_groups": 10931 },932 "placeholders_exact": {933 "mean": 1.0,934 "ci_low": 1.0,935 "ci_high": 1.0,936 "n_items": 69,937 "n_groups": 10938 }939 },940 "discussion": {941 "n": 103,942 "chrf": {943 "mean": 69.79505322674632,944 "ci_low": 67.58213701186176,945 "ci_high": 72.12885141279587,946 "n_items": 103,947 "n_groups": 14948 },949 "edit_sim": {950 "mean": 0.6767096913363885,951 "ci_low": 0.6486507406392065,952 "ci_high": 0.7073750896730979,953 "n_items": 103,954 "n_groups": 14955 },956 "rougeL": {957 "mean": 0.6819532385969715,958 "ci_low": 0.6476656852737122,959 "ci_high": 0.7155599705679301,960 "n_items": 103,961 "n_groups": 14962 },963 "exact_match": {964 "mean": 0.0,965 "ci_low": 0.0,966 "ci_high": 0.0,967 "n_items": 103,968 "n_groups": 14969 },970 "number_recall": {971 "mean": 1.0,972 "ci_low": 1.0,973 "ci_high": 1.0,974 "n_items": 103,975 "n_groups": 14976 },977 "placeholders_exact": {978 "mean": 1.0,979 "ci_low": 1.0,980 "ci_high": 1.0,981 "n_items": 103,982 "n_groups": 14983 }984 },985 "introduction": {986 "n": 99,987 "chrf": {988 "mean": 68.64066741335601,989 "ci_low": 65.52811690414141,990 "ci_high": 71.48418224133398,991 "n_items": 99,992 "n_groups": 14993 },994 "edit_sim": {995 "mean": 0.6398276947222239,996 "ci_low": 0.6012620452652253,997 "ci_high": 0.675716644902181,998 "n_items": 99,999 "n_groups": 141000 },1001 "rougeL": {1002 "mean": 0.6456982866997059,1003 "ci_low": 0.6100090541423175,1004 "ci_high": 0.6785689764557162,1005 "n_items": 99,1006 "n_groups": 141007 },1008 "exact_match": {1009 "mean": 0.0,1010 "ci_low": 0.0,1011 "ci_high": 0.0,1012 "n_items": 99,1013 "n_groups": 141014 },1015 "number_recall": {1016 "mean": 1.0,1017 "ci_low": 1.0,1018 "ci_high": 1.0,1019 "n_items": 99,1020 "n_groups": 141021 },1022 "placeholders_exact": {1023 "mean": 1.0,1024 "ci_low": 1.0,1025 "ci_high": 1.0,1026 "n_items": 99,1027 "n_groups": 141028 }1029 },1030 "limitations": {1031 "n": 42,1032 "chrf": {1033 "mean": 68.65003355158368,1034 "ci_low": 65.54931042154004,1035 "ci_high": 71.45110591361977,1036 "n_items": 42,1037 "n_groups": 41038 },1039 "edit_sim": {1040 "mean": 0.6493116276096962,1041 "ci_low": 0.6087002402299083,1042 "ci_high": 0.6910383117715111,1043 "n_items": 42,1044 "n_groups": 41045 },1046 "rougeL": {1047 "mean": 0.6527845991895832,1048 "ci_low": 0.6256633533558272,1049 "ci_high": 0.6889462603012578,1050 "n_items": 42,1051 "n_groups": 41052 },1053 "exact_match": {1054 "mean": 0.0,1055 "ci_low": 0.0,1056 "ci_high": 0.0,1057 "n_items": 42,1058 "n_groups": 41059 },1060 "number_recall": {1061 "mean": 1.0,1062 "ci_low": 1.0,1063 "ci_high": 1.0,1064 "n_items": 42,1065 "n_groups": 41066 },1067 "placeholders_exact": {1068 "mean": 1.0,1069 "ci_low": 1.0,1070 "ci_high": 1.0,1071 "n_items": 42,1072 "n_groups": 41073 }1074 },1075 "methods": {1076 "n": 98,1077 "chrf": {1078 "mean": 69.99308015556605,1079 "ci_low": 65.97817878199577,1080 "ci_high": 74.42243302357211,1081 "n_items": 98,1082 "n_groups": 121083 },1084 "edit_sim": {1085 "mean": 0.6666142668383377,1086 "ci_low": 0.641320121570484,1087 "ci_high": 0.6967223039834705,1088 "n_items": 98,1089 "n_groups": 121090 },1091 "rougeL": {1092 "mean": 0.676345893465207,1093 "ci_low": 0.6404653517648197,1094 "ci_high": 0.716263432801506,1095 "n_items": 98,1096 "n_groups": 121097 },1098 "exact_match": {1099 "mean": 0.0,1100 "ci_low": 0.0,1101 "ci_high": 0.0,1102 "n_items": 98,1103 "n_groups": 121104 },1105 "number_recall": {1106 "mean": 1.0,1107 "ci_low": 1.0,1108 "ci_high": 1.0,1109 "n_items": 98,1110 "n_groups": 121111 },1112 "placeholders_exact": {1113 "mean": 1.0,1114 "ci_low": 1.0,1115 "ci_high": 1.0,1116 "n_items": 98,1117 "n_groups": 121118 }1119 },1120 "other": {1121 "n": 70,1122 "chrf": {1123 "mean": 68.61172828728415,1124 "ci_low": 63.66995372759833,1125 "ci_high": 72.53507576803696,1126 "n_items": 70,1127 "n_groups": 91128 },1129 "edit_sim": {1130 "mean": 0.655169089536022,1131 "ci_low": 0.6166021682587611,1132 "ci_high": 0.6869100788906694,1133 "n_items": 70,1134 "n_groups": 91135 },1136 "rougeL": {1137 "mean": 0.6522792001917879,1138 "ci_low": 0.5961932918023273,1139 "ci_high": 0.6973985313314132,1140 "n_items": 70,1141 "n_groups": 91142 },1143 "exact_match": {1144 "mean": 0.0,1145 "ci_low": 0.0,1146 "ci_high": 0.0,1147 "n_items": 70,1148 "n_groups": 91149 },1150 "number_recall": {1151 "mean": 1.0,1152 "ci_low": 1.0,1153 "ci_high": 1.0,1154 "n_items": 70,1155 "n_groups": 91156 },1157 "placeholders_exact": {1158 "mean": 1.0,1159 "ci_low": 1.0,1160 "ci_high": 1.0,1161 "n_items": 70,1162 "n_groups": 91163 }1164 },1165 "related_work": {1166 "n": 6,1167 "chrf": {1168 "mean": 72.92571178933999,1169 "ci_low": 72.92571178933999,1170 "ci_high": 72.92571178933999,1171 "n_items": 6,1172 "n_groups": 11173 },1174 "edit_sim": {1175 "mean": 0.7079664511827871,1176 "ci_low": 0.7079664511827871,1177 "ci_high": 0.7079664511827871,1178 "n_items": 6,1179 "n_groups": 11180 },1181 "rougeL": {1182 "mean": 0.7254052185421385,1183 "ci_low": 0.7254052185421385,1184 "ci_high": 0.7254052185421385,1185 "n_items": 6,1186 "n_groups": 11187 },1188 "exact_match": {1189 "mean": 0.0,1190 "ci_low": 0.0,1191 "ci_high": 0.0,1192 "n_items": 6,1193 "n_groups": 11194 },1195 "number_recall": {1196 "mean": 1.0,1197 "ci_low": 1.0,1198 "ci_high": 1.0,1199 "n_items": 6,1200 "n_groups": 11201 },1202 "placeholders_exact": {1203 "mean": 1.0,1204 "ci_low": 1.0,1205 "ci_high": 1.0,1206 "n_items": 6,1207 "n_groups": 11208 }1209 },1210 "results": {1211 "n": 78,1212 "chrf": {1213 "mean": 69.08260217311539,1214 "ci_low": 67.45871643766075,1215 "ci_high": 70.79613802878727,1216 "n_items": 78,1217 "n_groups": 91218 },1219 "edit_sim": {1220 "mean": 0.6344507949794322,1221 "ci_low": 0.5993126506410352,1222 "ci_high": 0.6654409703827433,1223 "n_items": 78,1224 "n_groups": 91225 },1226 "rougeL": {1227 "mean": 0.665232314491607,1228 "ci_low": 0.6416590048235266,1229 "ci_high": 0.6851205022360017,1230 "n_items": 78,1231 "n_groups": 91232 },1233 "exact_match": {1234 "mean": 0.0,1235 "ci_low": 0.0,1236 "ci_high": 0.0,1237 "n_items": 78,1238 "n_groups": 91239 },1240 "number_recall": {1241 "mean": 1.0,1242 "ci_low": 1.0,1243 "ci_high": 1.0,1244 "n_items": 78,1245 "n_groups": 91246 },1247 "placeholders_exact": {1248 "mean": 1.0,1249 "ci_low": 1.0,1250 "ci_high": 1.0,1251 "n_items": 78,1252 "n_groups": 91253 }1254 },1255 "theory": {1256 "n": 59,1257 "chrf": {1258 "mean": 71.34826842685291,1259 "ci_low": 69.66533324121652,1260 "ci_high": 73.47815040062244,1261 "n_items": 59,1262 "n_groups": 61263 },1264 "edit_sim": {1265 "mean": 0.6845207455244317,1266 "ci_low": 0.6596864918654889,1267 "ci_high": 0.7020953531715937,1268 "n_items": 59,1269 "n_groups": 61270 },1271 "rougeL": {1272 "mean": 0.6915566901646324,1273 "ci_low": 0.671790791725937,1274 "ci_high": 0.7097986509207165,1275 "n_items": 59,1276 "n_groups": 61277 },1278 "exact_match": {1279 "mean": 0.0,1280 "ci_low": 0.0,1281 "ci_high": 0.0,1282 "n_items": 59,1283 "n_groups": 61284 },1285 "number_recall": {1286 "mean": 1.0,1287 "ci_low": 1.0,1288 "ci_high": 1.0,1289 "n_items": 59,1290 "n_groups": 61291 },1292 "placeholders_exact": {1293 "mean": 1.0,1294 "ci_low": 1.0,1295 "ci_high": 1.0,1296 "n_items": 59,1297 "n_groups": 61298 }1299 }1300 }1301 },1302 "B1_generic_paraphrase": {1303 "overall": {1304 "n": 812,1305 "chrf": {1306 "mean": 67.22281862270368,1307 "ci_low": 66.19548639547516,1308 "ci_high": 68.45116055136845,1309 "n_items": 812,1310 "n_groups": 541311 },1312 "edit_sim": {1313 "mean": 0.5616042313764219,1314 "ci_low": 0.5505538258982354,1315 "ci_high": 0.5728548759199493,1316 "n_items": 812,1317 "n_groups": 541318 },1319 "rougeL": {1320 "mean": 0.5964578799608277,1321 "ci_low": 0.5845265532261953,1322 "ci_high": 0.6104448248142234,1323 "n_items": 812,1324 "n_groups": 541325 },1326 "exact_match": {1327 "mean": 0.0,1328 "ci_low": 0.0,1329 "ci_high": 0.0,1330 "n_items": 812,1331 "n_groups": 541332 },1333 "number_recall": {1334 "mean": 0.9990763546798029,1335 "ci_low": 0.9974460839954598,1336 "ci_high": 1.0,1337 "n_items": 812,1338 "n_groups": 541339 },1340 "placeholders_exact": {1341 "mean": 1.0,1342 "ci_low": 1.0,1343 "ci_high": 1.0,1344 "n_items": 812,1345 "n_groups": 541346 }1347 },1348 "by_variant": {1349 "A": {1350 "n": 133,1351 "chrf": {1352 "mean": 65.30502569789257,1353 "ci_low": 64.21210239583809,1354 "ci_high": 66.51978908955616,1355 "n_items": 133,1356 "n_groups": 501357 },1358 "edit_sim": {1359 "mean": 0.5187303374413998,1360 "ci_low": 0.5028219953576125,1361 "ci_high": 0.5337049452227687,1362 "n_items": 133,1363 "n_groups": 501364 },1365 "rougeL": {1366 "mean": 0.5513946028503427,1367 "ci_low": 0.5373466310434943,1368 "ci_high": 0.5659102795356726,1369 "n_items": 133,1370 "n_groups": 501371 },1372 "exact_match": {1373 "mean": 0.0,1374 "ci_low": 0.0,1375 "ci_high": 0.0,1376 "n_items": 133,1377 "n_groups": 501378 },1379 "number_recall": {1380 "mean": 1.0,1381 "ci_low": 1.0,1382 "ci_high": 1.0,1383 "n_items": 133,1384 "n_groups": 501385 },1386 "placeholders_exact": {1387 "mean": 1.0,1388 "ci_low": 1.0,1389 "ci_high": 1.0,1390 "n_items": 133,1391 "n_groups": 501392 }1393 },1394 "B": {1395 "n": 136,1396 "chrf": {1397 "mean": 74.17275136280611,1398 "ci_low": 72.8457931621682,1399 "ci_high": 75.60817713315761,1400 "n_items": 136,1401 "n_groups": 521402 },1403 "edit_sim": {1404 "mean": 0.6405443357448881,1405 "ci_low": 0.623190895106162,1406 "ci_high": 0.6594598881936811,1407 "n_items": 136,1408 "n_groups": 521409 },1410 "rougeL": {1411 "mean": 0.686192932537256,1412 "ci_low": 0.668613796555602,1413 "ci_high": 0.7052401780291054,1414 "n_items": 136,1415 "n_groups": 521416 },1417 "exact_match": {1418 "mean": 0.0,1419 "ci_low": 0.0,1420 "ci_high": 0.0,1421 "n_items": 136,1422 "n_groups": 521423 },1424 "number_recall": {1425 "mean": 1.0,1426 "ci_low": 1.0,1427 "ci_high": 1.0,1428 "n_items": 136,1429 "n_groups": 521430 },1431 "placeholders_exact": {1432 "mean": 1.0,1433 "ci_low": 1.0,1434 "ci_high": 1.0,1435 "n_items": 136,1436 "n_groups": 521437 }1438 },1439 "C": {1440 "n": 135,1441 "chrf": {1442 "mean": 69.85147895940817,1443 "ci_low": 68.55908041391454,1444 "ci_high": 71.43530975338015,1445 "n_items": 135,1446 "n_groups": 531447 },1448 "edit_sim": {1449 "mean": 0.61851539852437,1450 "ci_low": 0.6015981828374032,1451 "ci_high": 0.6372051876687906,1452 "n_items": 135,1453 "n_groups": 531454 },1455 "rougeL": {1456 "mean": 0.6643184461601656,1457 "ci_low": 0.6476719298846596,1458 "ci_high": 0.6838065330437383,1459 "n_items": 135,1460 "n_groups": 531461 },1462 "exact_match": {1463 "mean": 0.0,1464 "ci_low": 0.0,1465 "ci_high": 0.0,1466 "n_items": 135,1467 "n_groups": 531468 },1469 "number_recall": {1470 "mean": 0.9981481481481481,1471 "ci_low": 0.994861904677408,1472 "ci_high": 1.0,1473 "n_items": 135,1474 "n_groups": 531475 },1476 "placeholders_exact": {1477 "mean": 1.0,1478 "ci_low": 1.0,1479 "ci_high": 1.0,1480 "n_items": 135,1481 "n_groups": 531482 }1483 },1484 "D": {1485 "n": 136,1486 "chrf": {1487 "mean": 63.70764827205955,1488 "ci_low": 62.61841309780673,1489 "ci_high": 64.9503140624043,1490 "n_items": 136,1491 "n_groups": 511492 },1493 "edit_sim": {1494 "mean": 0.5137832081902536,1495 "ci_low": 0.497374491500868,1496 "ci_high": 0.5322395651418722,1497 "n_items": 136,1498 "n_groups": 511499 },1500 "rougeL": {1501 "mean": 0.5433131569517999,1502 "ci_low": 0.5274106541518437,1503 "ci_high": 0.5607624424490242,1504 "n_items": 136,1505 "n_groups": 511506 },1507 "exact_match": {1508 "mean": 0.0,1509 "ci_low": 0.0,1510 "ci_high": 0.0,1511 "n_items": 136,1512 "n_groups": 511513 },1514 "number_recall": {1515 "mean": 1.0,1516 "ci_low": 1.0,1517 "ci_high": 1.0,1518 "n_items": 136,1519 "n_groups": 511520 },1521 "placeholders_exact": {1522 "mean": 1.0,1523 "ci_low": 1.0,1524 "ci_high": 1.0,1525 "n_items": 136,1526 "n_groups": 511527 }1528 },1529 "E": {1530 "n": 272,1531 "chrf": {1532 "mean": 65.13851578659502,1533 "ci_low": 64.08597386333577,1534 "ci_high": 66.35871963126893,1535 "n_items": 272,1536 "n_groups": 521537 },1538 "edit_sim": {1539 "mean": 0.5387624125808057,1540 "ci_low": 0.5238365396313515,1541 "ci_high": 0.5525492854422089,1542 "n_items": 272,1543 "n_groups": 521544 },1545 "rougeL": {1546 "mean": 0.5665165365697153,1547 "ci_low": 0.5523750893823125,1548 "ci_high": 0.5826646816302332,1549 "n_items": 272,1550 "n_groups": 521551 },1552 "exact_match": {1553 "mean": 0.0,1554 "ci_low": 0.0,1555 "ci_high": 0.0,1556 "n_items": 272,1557 "n_groups": 521558 },1559 "number_recall": {1560 "mean": 0.9981617647058824,1561 "ci_low": 0.9950121571072319,1562 "ci_high": 1.0,1563 "n_items": 272,1564 "n_groups": 521565 },1566 "placeholders_exact": {1567 "mean": 1.0,1568 "ci_low": 1.0,1569 "ci_high": 1.0,1570 "n_items": 272,1571 "n_groups": 521572 }1573 }1574 },1575 "by_discipline": {1576 "biomedicine": {1577 "n": 380,1578 "chrf": {1579 "mean": 67.70593358586379,1580 "ci_low": 66.45106616159777,1581 "ci_high": 69.04450625435487,1582 "n_items": 380,1583 "n_groups": 311584 },1585 "edit_sim": {1586 "mean": 0.5606729922019881,1587 "ci_low": 0.5444398216003397,1588 "ci_high": 0.5760364108374081,1589 "n_items": 380,1590 "n_groups": 311591 },1592 "rougeL": {1593 "mean": 0.5965037270031164,1594 "ci_low": 0.5806783791451727,1595 "ci_high": 0.6103060975132857,1596 "n_items": 380,1597 "n_groups": 311598 },1599 "exact_match": {1600 "mean": 0.0,1601 "ci_low": 0.0,1602 "ci_high": 0.0,1603 "n_items": 380,1604 "n_groups": 311605 },1606 "number_recall": {1607 "mean": 1.0,1608 "ci_low": 1.0,1609 "ci_high": 1.0,1610 "n_items": 380,1611 "n_groups": 311612 },1613 "placeholders_exact": {1614 "mean": 1.0,1615 "ci_low": 1.0,1616 "ci_high": 1.0,1617 "n_items": 380,1618 "n_groups": 311619 }1620 },1621 "engineering": {1622 "n": 12,1623 "chrf": {1624 "mean": 70.22404126869381,1625 "ci_low": 69.48811018656174,1626 "ci_high": 70.95997235082588,1627 "n_items": 12,1628 "n_groups": 21629 },1630 "edit_sim": {1631 "mean": 0.6255444370972774,1632 "ci_low": 0.613507472718955,1633 "ci_high": 0.6375814014755997,1634 "n_items": 12,1635 "n_groups": 21636 },1637 "rougeL": {1638 "mean": 0.6331469330861254,1639 "ci_low": 0.6283510121062813,1640 "ci_high": 0.6379428540659693,1641 "n_items": 12,1642 "n_groups": 21643 },1644 "exact_match": {1645 "mean": 0.0,1646 "ci_low": 0.0,1647 "ci_high": 0.0,1648 "n_items": 12,1649 "n_groups": 21650 },1651 "number_recall": {1652 "mean": 1.0,1653 "ci_low": 1.0,1654 "ci_high": 1.0,1655 "n_items": 12,1656 "n_groups": 21657 },1658 "placeholders_exact": {1659 "mean": 1.0,1660 "ci_low": 1.0,1661 "ci_high": 1.0,1662 "n_items": 12,1663 "n_groups": 21664 }1665 },1666 "finance": {1667 "n": 96,1668 "chrf": {1669 "mean": 66.03363225494721,1670 "ci_low": 66.03363225494721,1671 "ci_high": 66.03363225494721,1672 "n_items": 96,1673 "n_groups": 11674 },1675 "edit_sim": {1676 "mean": 0.5576237334121162,1677 "ci_low": 0.5576237334121162,1678 "ci_high": 0.5576237334121162,1679 "n_items": 96,1680 "n_groups": 11681 },1682 "rougeL": {1683 "mean": 0.5775220063986155,1684 "ci_low": 0.5775220063986155,1685 "ci_high": 0.5775220063986155,1686 "n_items": 96,1687 "n_groups": 11688 },1689 "exact_match": {1690 "mean": 0.0,1691 "ci_low": 0.0,1692 "ci_high": 0.0,1693 "n_items": 96,1694 "n_groups": 11695 },1696 "number_recall": {1697 "mean": 0.9921875,1698 "ci_low": 0.9921875,1699 "ci_high": 0.9921875,1700 "n_items": 96,1701 "n_groups": 11702 },1703 "placeholders_exact": {1704 "mean": 1.0,1705 "ci_low": 1.0,1706 "ci_high": 1.0,1707 "n_items": 96,1708 "n_groups": 11709 }1710 },1711 "life_sciences": {1712 "n": 162,1713 "chrf": {1714 "mean": 69.06530590101407,1715 "ci_low": 66.1259617003405,1716 "ci_high": 71.7415362569236,1717 "n_items": 162,1718 "n_groups": 151719 },1720 "edit_sim": {1721 "mean": 0.5654660118408845,1722 "ci_low": 0.5420929335585003,1723 "ci_high": 0.5855898734980394,1724 "n_items": 162,1725 "n_groups": 151726 },1727 "rougeL": {1728 "mean": 0.6208622272273819,1729 "ci_low": 0.5932686059262908,1730 "ci_high": 0.6464883944867964,1731 "n_items": 162,1732 "n_groups": 151733 },1734 "exact_match": {1735 "mean": 0.0,1736 "ci_low": 0.0,1737 "ci_high": 0.0,1738 "n_items": 162,1739 "n_groups": 151740 },1741 "number_recall": {1742 "mean": 1.0,1743 "ci_low": 1.0,1744 "ci_high": 1.0,1745 "n_items": 162,1746 "n_groups": 151747 },1748 "placeholders_exact": {1749 "mean": 1.0,1750 "ci_low": 1.0,1751 "ci_high": 1.0,1752 "n_items": 162,1753 "n_groups": 151754 }1755 },1756 "mathematics": {1757 "n": 60,1758 "chrf": {1759 "mean": 65.0405960258002,1760 "ci_low": 65.0405960258002,1761 "ci_high": 65.0405960258002,1762 "n_items": 60,1763 "n_groups": 11764 },1765 "edit_sim": {1766 "mean": 0.5746013210418329,1767 "ci_low": 0.5746013210418329,1768 "ci_high": 0.5746013210418329,1769 "n_items": 60,1770 "n_groups": 11771 },1772 "rougeL": {1773 "mean": 0.5819507631846043,1774 "ci_low": 0.5819507631846043,1775 "ci_high": 0.5819507631846043,1776 "n_items": 60,1777 "n_groups": 11778 },1779 "exact_match": {1780 "mean": 0.0,1781 "ci_low": 0.0,1782 "ci_high": 0.0,1783 "n_items": 60,1784 "n_groups": 11785 },1786 "number_recall": {1787 "mean": 1.0,1788 "ci_low": 1.0,1789 "ci_high": 1.0,1790 "n_items": 60,1791 "n_groups": 11792 },1793 "placeholders_exact": {1794 "mean": 1.0,1795 "ci_low": 1.0,1796 "ci_high": 1.0,1797 "n_items": 60,1798 "n_groups": 11799 }1800 },1801 "social_sciences": {1802 "n": 30,1803 "chrf": {1804 "mean": 66.2811056076223,1805 "ci_low": 53.90461520982321,1806 "ci_high": 69.37522820707206,1807 "n_items": 30,1808 "n_groups": 21809 },1810 "edit_sim": {1811 "mean": 0.6078321721494134,1812 "ci_low": 0.5414236469284833,1813 "ci_high": 0.624434303454646,1814 "n_items": 30,1815 "n_groups": 21816 },1817 "rougeL": {1818 "mean": 0.6162431447945512,1819 "ci_low": 0.44565263918246273,1820 "ci_high": 0.6588907711975734,1821 "n_items": 30,1822 "n_groups": 21823 },1824 "exact_match": {1825 "mean": 0.0,1826 "ci_low": 0.0,1827 "ci_high": 0.0,1828 "n_items": 30,1829 "n_groups": 21830 },1831 "number_recall": {1832 "mean": 1.0,1833 "ci_low": 1.0,1834 "ci_high": 1.0,1835 "n_items": 30,1836 "n_groups": 21837 },1838 "placeholders_exact": {1839 "mean": 1.0,1840 "ci_low": 1.0,1841 "ci_high": 1.0,1842 "n_items": 30,1843 "n_groups": 21844 }1845 },1846 "statistics": {1847 "n": 72,1848 "chrf": {1849 "mean": 63.8237261328741,1850 "ci_low": 63.462284783198186,1851 "ci_high": 64.00444680771206,1852 "n_items": 72,1853 "n_groups": 21854 },1855 "edit_sim": {1856 "mean": 0.5223881783743463,1857 "ci_low": 0.5161922272184185,1858 "ci_high": 0.5347800806862014,1859 "n_items": 72,1860 "n_groups": 21861 },1862 "rougeL": {1863 "mean": 0.5642845209716474,1864 "ci_low": 0.5484689113318404,1865 "ci_high": 0.5959157402512614,1866 "n_items": 72,1867 "n_groups": 21868 },1869 "exact_match": {1870 "mean": 0.0,1871 "ci_low": 0.0,1872 "ci_high": 0.0,1873 "n_items": 72,1874 "n_groups": 21875 },1876 "number_recall": {1877 "mean": 1.0,1878 "ci_low": 1.0,1879 "ci_high": 1.0,1880 "n_items": 72,1881 "n_groups": 21882 },1883 "placeholders_exact": {1884 "mean": 1.0,1885 "ci_low": 1.0,1886 "ci_high": 1.0,1887 "n_items": 72,1888 "n_groups": 21889 }1890 }1891 },1892 "by_era": {1893 "1980-1999": {1894 "n": 8,1895 "chrf": {1896 "mean": 71.37413161008944,1897 "ci_low": 69.31548283255509,1898 "ci_high": 74.80521290598,1899 "n_items": 8,1900 "n_groups": 21901 },1902 "edit_sim": {1903 "mean": 0.6531459870747707,1904 "ci_low": 0.6077169799263029,1905 "ci_high": 0.7288609989888838,1906 "n_items": 8,1907 "n_groups": 21908 },1909 "rougeL": {1910 "mean": 0.6710787004207134,1911 "ci_low": 0.6290555992843334,1912 "ci_high": 0.74111720231468,1913 "n_items": 8,1914 "n_groups": 21915 },1916 "exact_match": {1917 "mean": 0.0,1918 "ci_low": 0.0,1919 "ci_high": 0.0,1920 "n_items": 8,1921 "n_groups": 21922 },1923 "number_recall": {1924 "mean": 1.0,1925 "ci_low": 1.0,1926 "ci_high": 1.0,1927 "n_items": 8,1928 "n_groups": 21929 },1930 "placeholders_exact": {1931 "mean": 1.0,1932 "ci_low": 1.0,1933 "ci_high": 1.0,1934 "n_items": 8,1935 "n_groups": 21936 }1937 },1938 "2000-2009": {1939 "n": 89,1940 "chrf": {1941 "mean": 66.81830278263894,1942 "ci_low": 64.62363098835701,1943 "ci_high": 70.35141648891873,1944 "n_items": 89,1945 "n_groups": 71946 },1947 "edit_sim": {1948 "mean": 0.5493974362549647,1949 "ci_low": 0.5196199159532715,1950 "ci_high": 0.5772540695348447,1951 "n_items": 89,1952 "n_groups": 71953 },1954 "rougeL": {1955 "mean": 0.6003431704591587,1956 "ci_low": 0.5807407645541357,1957 "ci_high": 0.6298043056784458,1958 "n_items": 89,1959 "n_groups": 71960 },1961 "exact_match": {1962 "mean": 0.0,1963 "ci_low": 0.0,1964 "ci_high": 0.0,1965 "n_items": 89,1966 "n_groups": 71967 },1968 "number_recall": {1969 "mean": 1.0,1970 "ci_low": 1.0,1971 "ci_high": 1.0,1972 "n_items": 89,1973 "n_groups": 71974 },1975 "placeholders_exact": {1976 "mean": 1.0,1977 "ci_low": 1.0,1978 "ci_high": 1.0,1979 "n_items": 89,1980 "n_groups": 71981 }1982 },1983 "2010-2017": {1984 "n": 715,1985 "chrf": {1986 "mean": 67.2267226868529,1987 "ci_low": 66.11867735735208,1988 "ci_high": 68.53720209953407,1989 "n_items": 715,1990 "n_groups": 451991 },1992 "edit_sim": {1993 "mean": 0.5620994351809294,1994 "ci_low": 0.5497565538506669,1995 "ci_high": 0.57472638966721,1996 "n_items": 715,1997 "n_groups": 451998 },1999 "rougeL": {2000 "mean": 0.5951393381174283,2001 "ci_low": 0.581756609093635,2002 "ci_high": 0.6104659403073984,2003 "n_items": 715,2004 "n_groups": 452005 },2006 "exact_match": {2007 "mean": 0.0,2008 "ci_low": 0.0,2009 "ci_high": 0.0,2010 "n_items": 715,2011 "n_groups": 452012 },2013 "number_recall": {2014 "mean": 0.9989510489510489,2015 "ci_low": 0.9970433153978338,2016 "ci_high": 1.0,2017 "n_items": 715,2018 "n_groups": 452019 },2020 "placeholders_exact": {2021 "mean": 1.0,2022 "ci_low": 1.0,2023 "ci_high": 1.0,2024 "n_items": 715,2025 "n_groups": 452026 }2027 }2028 },2029 "by_role": {2030 "abstract": {2031 "n": 83,2032 "chrf": {2033 "mean": 70.11467473449797,2034 "ci_low": 68.47549762416939,2035 "ci_high": 71.65096061546517,2036 "n_items": 83,2037 "n_groups": 152038 },2039 "edit_sim": {2040 "mean": 0.5969785002608264,2041 "ci_low": 0.5655089967807777,2042 "ci_high": 0.6262723296047464,2043 "n_items": 83,2044 "n_groups": 152045 },2046 "rougeL": {2047 "mean": 0.6337447997406698,2048 "ci_low": 0.6059864995284362,2049 "ci_high": 0.6597235316363087,2050 "n_items": 83,2051 "n_groups": 152052 },2053 "exact_match": {2054 "mean": 0.0,2055 "ci_low": 0.0,2056 "ci_high": 0.0,2057 "n_items": 83,2058 "n_groups": 152059 },2060 "number_recall": {2061 "mean": 1.0,2062 "ci_low": 1.0,2063 "ci_high": 1.0,2064 "n_items": 83,2065 "n_groups": 152066 },2067 "placeholders_exact": {2068 "mean": 1.0,2069 "ci_low": 1.0,2070 "ci_high": 1.0,2071 "n_items": 83,2072 "n_groups": 152073 }2074 },2075 "background": {2076 "n": 58,2077 "chrf": {2078 "mean": 67.15329297308348,2079 "ci_low": 64.52070359820037,2080 "ci_high": 69.50365566001302,2081 "n_items": 58,2082 "n_groups": 72083 },2084 "edit_sim": {2085 "mean": 0.5519112885151441,2086 "ci_low": 0.5276604141206771,2087 "ci_high": 0.5867680988711513,2088 "n_items": 58,2089 "n_groups": 72090 },2091 "rougeL": {2092 "mean": 0.5852992344296633,2093 "ci_low": 0.5622374986089504,2094 "ci_high": 0.6102505420796832,2095 "n_items": 58,2096 "n_groups": 72097 },2098 "exact_match": {2099 "mean": 0.0,2100 "ci_low": 0.0,2101 "ci_high": 0.0,2102 "n_items": 58,2103 "n_groups": 72104 },2105 "number_recall": {2106 "mean": 1.0,2107 "ci_low": 1.0,2108 "ci_high": 1.0,2109 "n_items": 58,2110 "n_groups": 72111 },2112 "placeholders_exact": {2113 "mean": 1.0,2114 "ci_low": 1.0,2115 "ci_high": 1.0,2116 "n_items": 58,2117 "n_groups": 72118 }2119 },2120 "conclusion": {2121 "n": 47,2122 "chrf": {2123 "mean": 64.62937712663872,2124 "ci_low": 62.97358932592427,2125 "ci_high": 66.10651998968764,2126 "n_items": 47,2127 "n_groups": 82128 },2129 "edit_sim": {2130 "mean": 0.5326807373384927,2131 "ci_low": 0.5158929749283794,2132 "ci_high": 0.550505915753096,2133 "n_items": 47,2134 "n_groups": 82135 },2136 "rougeL": {2137 "mean": 0.551862453941936,2138 "ci_low": 0.5281221822514791,2139 "ci_high": 0.5709997478170884,2140 "n_items": 47,2141 "n_groups": 82142 },2143 "exact_match": {2144 "mean": 0.0,2145 "ci_low": 0.0,2146 "ci_high": 0.0,2147 "n_items": 47,2148 "n_groups": 82149 },2150 "number_recall": {2151 "mean": 1.0,2152 "ci_low": 1.0,2153 "ci_high": 1.0,2154 "n_items": 47,2155 "n_groups": 82156 },2157 "placeholders_exact": {2158 "mean": 1.0,2159 "ci_low": 1.0,2160 "ci_high": 1.0,2161 "n_items": 47,2162 "n_groups": 82163 }2164 },2165 "data": {2166 "n": 69,2167 "chrf": {2168 "mean": 69.2302051566615,2169 "ci_low": 65.869252251384,2170 "ci_high": 73.08935017790813,2171 "n_items": 69,2172 "n_groups": 102173 },2174 "edit_sim": {2175 "mean": 0.5477136377142539,2176 "ci_low": 0.5063933403889598,2177 "ci_high": 0.5952795136043184,2178 "n_items": 69,2179 "n_groups": 102180 },2181 "rougeL": {2182 "mean": 0.6065446400269282,2183 "ci_low": 0.5663080730447437,2184 "ci_high": 0.658083524262669,2185 "n_items": 69,2186 "n_groups": 102187 },2188 "exact_match": {2189 "mean": 0.0,2190 "ci_low": 0.0,2191 "ci_high": 0.0,2192 "n_items": 69,2193 "n_groups": 102194 },2195 "number_recall": {2196 "mean": 0.9891304347826086,2197 "ci_low": 0.97,2198 "ci_high": 1.0,2199 "n_items": 69,2200 "n_groups": 102201 },2202 "placeholders_exact": {2203 "mean": 1.0,2204 "ci_low": 1.0,2205 "ci_high": 1.0,2206 "n_items": 69,2207 "n_groups": 102208 }2209 },2210 "discussion": {2211 "n": 103,2212 "chrf": {2213 "mean": 66.75693992586842,2214 "ci_low": 64.18266513890903,2215 "ci_high": 69.66376891924664,2216 "n_items": 103,2217 "n_groups": 142218 },2219 "edit_sim": {2220 "mean": 0.5616160507878681,2221 "ci_low": 0.5293838064456335,2222 "ci_high": 0.6013049939278234,2223 "n_items": 103,2224 "n_groups": 142225 },2226 "rougeL": {2227 "mean": 0.5924901863269884,2228 "ci_low": 0.5522708314149152,2229 "ci_high": 0.6378521391168736,2230 "n_items": 103,2231 "n_groups": 142232 },2233 "exact_match": {2234 "mean": 0.0,2235 "ci_low": 0.0,2236 "ci_high": 0.0,2237 "n_items": 103,2238 "n_groups": 142239 },2240 "number_recall": {2241 "mean": 1.0,2242 "ci_low": 1.0,2243 "ci_high": 1.0,2244 "n_items": 103,2245 "n_groups": 142246 },2247 "placeholders_exact": {2248 "mean": 1.0,2249 "ci_low": 1.0,2250 "ci_high": 1.0,2251 "n_items": 103,2252 "n_groups": 142253 }2254 },2255 "introduction": {2256 "n": 99,2257 "chrf": {2258 "mean": 66.34402769670457,2259 "ci_low": 63.450990256213366,2260 "ci_high": 68.98770247106212,2261 "n_items": 99,2262 "n_groups": 142263 },2264 "edit_sim": {2265 "mean": 0.5572718199432473,2266 "ci_low": 0.5232685636494325,2267 "ci_high": 0.5909021083706305,2268 "n_items": 99,2269 "n_groups": 142270 },2271 "rougeL": {2272 "mean": 0.5833414881837599,2273 "ci_low": 0.5523562749120482,2274 "ci_high": 0.611212920139623,2275 "n_items": 99,2276 "n_groups": 142277 },2278 "exact_match": {2279 "mean": 0.0,2280 "ci_low": 0.0,2281 "ci_high": 0.0,2282 "n_items": 99,2283 "n_groups": 142284 },2285 "number_recall": {2286 "mean": 1.0,2287 "ci_low": 1.0,2288 "ci_high": 1.0,2289 "n_items": 99,2290 "n_groups": 142291 },2292 "placeholders_exact": {2293 "mean": 1.0,2294 "ci_low": 1.0,2295 "ci_high": 1.0,2296 "n_items": 99,2297 "n_groups": 142298 }2299 },2300 "limitations": {2301 "n": 42,2302 "chrf": {2303 "mean": 64.52893655496061,2304 "ci_low": 61.57758948331472,2305 "ci_high": 66.94143120627898,2306 "n_items": 42,2307 "n_groups": 42308 },2309 "edit_sim": {2310 "mean": 0.5399317766832242,2311 "ci_low": 0.5160504169752166,2312 "ci_high": 0.5676661454219561,2313 "n_items": 42,2314 "n_groups": 42315 },2316 "rougeL": {2317 "mean": 0.5582627897666311,2318 "ci_low": 0.5420876150246836,2319 "ci_high": 0.579829689422561,2320 "n_items": 42,2321 "n_groups": 42322 },2323 "exact_match": {2324 "mean": 0.0,2325 "ci_low": 0.0,2326 "ci_high": 0.0,2327 "n_items": 42,2328 "n_groups": 42329 },2330 "number_recall": {2331 "mean": 1.0,2332 "ci_low": 1.0,2333 "ci_high": 1.0,2334 "n_items": 42,2335 "n_groups": 42336 },2337 "placeholders_exact": {2338 "mean": 1.0,2339 "ci_low": 1.0,2340 "ci_high": 1.0,2341 "n_items": 42,2342 "n_groups": 42343 }2344 },2345 "methods": {2346 "n": 98,2347 "chrf": {2348 "mean": 67.64618813503128,2349 "ci_low": 63.65490728549705,2350 "ci_high": 72.32893143873626,2351 "n_items": 98,2352 "n_groups": 122353 },2354 "edit_sim": {2355 "mean": 0.5608452806645421,2356 "ci_low": 0.53305233719303,2357 "ci_high": 0.5958344064588281,2358 "n_items": 98,2359 "n_groups": 122360 },2361 "rougeL": {2362 "mean": 0.6047604447256865,2363 "ci_low": 0.5703526890341657,2364 "ci_high": 0.6452344710927227,2365 "n_items": 98,2366 "n_groups": 122367 },2368 "exact_match": {2369 "mean": 0.0,2370 "ci_low": 0.0,2371 "ci_high": 0.0,2372 "n_items": 98,2373 "n_groups": 122374 },2375 "number_recall": {2376 "mean": 1.0,2377 "ci_low": 1.0,2378 "ci_high": 1.0,2379 "n_items": 98,2380 "n_groups": 122381 },2382 "placeholders_exact": {2383 "mean": 1.0,2384 "ci_low": 1.0,2385 "ci_high": 1.0,2386 "n_items": 98,2387 "n_groups": 122388 }2389 },2390 "other": {2391 "n": 70,2392 "chrf": {2393 "mean": 66.94423657903184,2394 "ci_low": 62.102176740417775,2395 "ci_high": 70.30907028720885,2396 "n_items": 70,2397 "n_groups": 92398 },2399 "edit_sim": {2400 "mean": 0.573797472003944,2401 "ci_low": 0.5517750250148776,2402 "ci_high": 0.591704436139265,2403 "n_items": 70,2404 "n_groups": 92405 },2406 "rougeL": {2407 "mean": 0.5929557330227743,2408 "ci_low": 0.5486842265256683,2409 "ci_high": 0.6250560904246313,2410 "n_items": 70,2411 "n_groups": 92412 },2413 "exact_match": {2414 "mean": 0.0,2415 "ci_low": 0.0,2416 "ci_high": 0.0,2417 "n_items": 70,2418 "n_groups": 92419 },2420 "number_recall": {2421 "mean": 1.0,2422 "ci_low": 1.0,2423 "ci_high": 1.0,2424 "n_items": 70,2425 "n_groups": 92426 },2427 "placeholders_exact": {2428 "mean": 1.0,2429 "ci_low": 1.0,2430 "ci_high": 1.0,2431 "n_items": 70,2432 "n_groups": 92433 }2434 },2435 "related_work": {2436 "n": 6,2437 "chrf": {2438 "mean": 69.48811018656173,2439 "ci_low": 69.48811018656173,2440 "ci_high": 69.48811018656173,2441 "n_items": 6,2442 "n_groups": 12443 },2444 "edit_sim": {2445 "mean": 0.613507472718955,2446 "ci_low": 0.613507472718955,2447 "ci_high": 0.613507472718955,2448 "n_items": 6,2449 "n_groups": 12450 },2451 "rougeL": {2452 "mean": 0.6379428540659693,2453 "ci_low": 0.6379428540659693,2454 "ci_high": 0.6379428540659693,2455 "n_items": 6,2456 "n_groups": 12457 },2458 "exact_match": {2459 "mean": 0.0,2460 "ci_low": 0.0,2461 "ci_high": 0.0,2462 "n_items": 6,2463 "n_groups": 12464 },2465 "number_recall": {2466 "mean": 1.0,2467 "ci_low": 1.0,2468 "ci_high": 1.0,2469 "n_items": 6,2470 "n_groups": 12471 },2472 "placeholders_exact": {2473 "mean": 1.0,2474 "ci_low": 1.0,2475 "ci_high": 1.0,2476 "n_items": 6,2477 "n_groups": 12478 }2479 },2480 "results": {2481 "n": 78,2482 "chrf": {2483 "mean": 65.75433447451303,2484 "ci_low": 63.575171001050066,2485 "ci_high": 68.22108164519878,2486 "n_items": 78,2487 "n_groups": 92488 },2489 "edit_sim": {2490 "mean": 0.5389501758440873,2491 "ci_low": 0.503630425044354,2492 "ci_high": 0.571684496690067,2493 "n_items": 78,2494 "n_groups": 92495 },2496 "rougeL": {2497 "mean": 0.5907471871364096,2498 "ci_low": 0.5583674654893763,2499 "ci_high": 0.6252864850846936,2500 "n_items": 78,2501 "n_groups": 92502 },2503 "exact_match": {2504 "mean": 0.0,2505 "ci_low": 0.0,2506 "ci_high": 0.0,2507 "n_items": 78,2508 "n_groups": 92509 },2510 "number_recall": {2511 "mean": 1.0,2512 "ci_low": 1.0,2513 "ci_high": 1.0,2514 "n_items": 78,2515 "n_groups": 92516 },2517 "placeholders_exact": {2518 "mean": 1.0,2519 "ci_low": 1.0,2520 "ci_high": 1.0,2521 "n_items": 78,2522 "n_groups": 92523 }2524 },2525 "theory": {2526 "n": 59,2527 "chrf": {2528 "mean": 68.48518996488689,2529 "ci_low": 66.10985725723765,2530 "ci_high": 70.32620643780079,2531 "n_items": 59,2532 "n_groups": 62533 },2534 "edit_sim": {2535 "mean": 0.5947968278161672,2536 "ci_low": 0.5477955884139479,2537 "ci_high": 0.6204777642788725,2538 "n_items": 59,2539 "n_groups": 62540 },2541 "rougeL": {2542 "mean": 0.6285222499635464,2543 "ci_low": 0.5849371156162312,2544 "ci_high": 0.652960974707486,2545 "n_items": 59,2546 "n_groups": 62547 },2548 "exact_match": {2549 "mean": 0.0,2550 "ci_low": 0.0,2551 "ci_high": 0.0,2552 "n_items": 59,2553 "n_groups": 62554 },2555 "number_recall": {2556 "mean": 1.0,2557 "ci_low": 1.0,2558 "ci_high": 1.0,2559 "n_items": 59,2560 "n_groups": 62561 },2562 "placeholders_exact": {2563 "mean": 1.0,2564 "ci_low": 1.0,2565 "ci_high": 1.0,2566 "n_items": 59,2567 "n_groups": 62568 }2569 }2570 }2571 },2572 "htr-m1-small": {2573 "overall": {2574 "n": 812,2575 "chrf": {2576 "mean": 70.2733757962643,2577 "ci_low": 69.40336806337294,2578 "ci_high": 71.29254358060957,2579 "n_items": 812,2580 "n_groups": 542581 },2582 "edit_sim": {2583 "mean": 0.6625522562244212,2584 "ci_low": 0.6512142125528823,2585 "ci_high": 0.674257385012514,2586 "n_items": 812,2587 "n_groups": 542588 },2589 "rougeL": {2590 "mean": 0.6752048662628412,2591 "ci_low": 0.6638717346671306,2592 "ci_high": 0.6876304167465209,2593 "n_items": 812,2594 "n_groups": 542595 },2596 "exact_match": {2597 "mean": 0.0,2598 "ci_low": 0.0,2599 "ci_high": 0.0,2600 "n_items": 812,2601 "n_groups": 542602 },2603 "number_recall": {2604 "mean": 0.9900073194205615,2605 "ci_low": 0.9818729081654742,2606 "ci_high": 0.9957358582811808,2607 "n_items": 812,2608 "n_groups": 542609 },2610 "placeholders_exact": {2611 "mean": 0.9938423645320197,2612 "ci_low": 0.9863013698630136,2613 "ci_high": 0.9989142531107138,2614 "n_items": 812,2615 "n_groups": 542616 }2617 },2618 "by_variant": {2619 "A": {2620 "n": 133,2621 "chrf": {2622 "mean": 68.81427426935143,2623 "ci_low": 67.90273712066005,2624 "ci_high": 69.76922849941434,2625 "n_items": 133,2626 "n_groups": 502627 },2628 "edit_sim": {2629 "mean": 0.6291525870559047,2630 "ci_low": 0.6155407537412069,2631 "ci_high": 0.6410076494972141,2632 "n_items": 133,2633 "n_groups": 502634 },2635 "rougeL": {2636 "mean": 0.642568060180303,2637 "ci_low": 0.6287868250949599,2638 "ci_high": 0.6564146223271136,2639 "n_items": 133,2640 "n_groups": 502641 },2642 "exact_match": {2643 "mean": 0.0,2644 "ci_low": 0.0,2645 "ci_high": 0.0,2646 "n_items": 133,2647 "n_groups": 502648 },2649 "number_recall": {2650 "mean": 0.9858824693312227,2651 "ci_low": 0.9684502039709489,2652 "ci_high": 0.9975074718706047,2653 "n_items": 133,2654 "n_groups": 502655 },2656 "placeholders_exact": {2657 "mean": 0.9849624060150376,2658 "ci_low": 0.9603940751217979,2659 "ci_high": 1.0,2660 "n_items": 133,2661 "n_groups": 502662 }2663 },2664 "B": {2665 "n": 136,2666 "chrf": {2667 "mean": 75.59577823275183,2668 "ci_low": 74.47674227088903,2669 "ci_high": 76.85490111483648,2670 "n_items": 136,2671 "n_groups": 522672 },2673 "edit_sim": {2674 "mean": 0.7406365139299114,2675 "ci_low": 0.7256061590186067,2676 "ci_high": 0.7538411657667536,2677 "n_items": 136,2678 "n_groups": 522679 },2680 "rougeL": {2681 "mean": 0.7474577354810111,2682 "ci_low": 0.7340669153711216,2683 "ci_high": 0.7620475887981124,2684 "n_items": 136,2685 "n_groups": 522686 },2687 "exact_match": {2688 "mean": 0.0,2689 "ci_low": 0.0,2690 "ci_high": 0.0,2691 "n_items": 136,2692 "n_groups": 522693 },2694 "number_recall": {2695 "mean": 0.9897690136660725,2696 "ci_low": 0.9770364837282882,2697 "ci_high": 0.9980194904874796,2698 "n_items": 136,2699 "n_groups": 522700 },2701 "placeholders_exact": {2702 "mean": 1.0,2703 "ci_low": 1.0,2704 "ci_high": 1.0,2705 "n_items": 136,2706 "n_groups": 522707 }2708 },2709 "C": {2710 "n": 135,2711 "chrf": {2712 "mean": 72.26725840630415,2713 "ci_low": 71.08946683523685,2714 "ci_high": 73.7205859804765,2715 "n_items": 135,2716 "n_groups": 532717 },2718 "edit_sim": {2719 "mean": 0.7162861044656896,2720 "ci_low": 0.70303660587202,2721 "ci_high": 0.7316502400361826,2722 "n_items": 135,2723 "n_groups": 532724 },2725 "rougeL": {2726 "mean": 0.734298748684231,2727 "ci_low": 0.7201027891452566,2728 "ci_high": 0.7509043409189443,2729 "n_items": 135,2730 "n_groups": 532731 },2732 "exact_match": {2733 "mean": 0.0,2734 "ci_low": 0.0,2735 "ci_high": 0.0,2736 "n_items": 135,2737 "n_groups": 532738 },2739 "number_recall": {2740 "mean": 0.9953056203056203,2741 "ci_low": 0.9888592131169801,2742 "ci_high": 0.9991666666666666,2743 "n_items": 135,2744 "n_groups": 532745 },2746 "placeholders_exact": {2747 "mean": 1.0,2748 "ci_low": 1.0,2749 "ci_high": 1.0,2750 "n_items": 135,2751 "n_groups": 532752 }2753 },2754 "D": {2755 "n": 136,2756 "chrf": {2757 "mean": 67.50877828425519,2758 "ci_low": 66.35113385839598,2759 "ci_high": 68.62760034915185,2760 "n_items": 136,2761 "n_groups": 512762 },2763 "edit_sim": {2764 "mean": 0.612361761191184,2765 "ci_low": 0.5964389673536034,2766 "ci_high": 0.6282825265673462,2767 "n_items": 136,2768 "n_groups": 512769 },2770 "rougeL": {2771 "mean": 0.6279577602307514,2772 "ci_low": 0.6131777733724189,2773 "ci_high": 0.6430475485660481,2774 "n_items": 136,2775 "n_groups": 512776 },2777 "exact_match": {2778 "mean": 0.0,2779 "ci_low": 0.0,2780 "ci_high": 0.0,2781 "n_items": 136,2782 "n_groups": 512783 },2784 "number_recall": {2785 "mean": 0.990978721033868,2786 "ci_low": 0.9782371794871796,2787 "ci_high": 0.9986008901865919,2788 "n_items": 136,2789 "n_groups": 512790 },2791 "placeholders_exact": {2792 "mean": 0.9852941176470589,2793 "ci_low": 0.96,2794 "ci_high": 1.0,2795 "n_items": 136,2796 "n_groups": 512797 }2798 },2799 "E": {2800 "n": 272,2801 "chrf": {2802 "mean": 68.71832021168697,2803 "ci_low": 67.76259576684869,2804 "ci_high": 69.81374843076203,2805 "n_items": 272,2806 "n_groups": 522807 },2808 "edit_sim": {2809 "mean": 0.6382674575605058,2810 "ci_low": 0.6241484327146328,2811 "ci_high": 0.6541700402494317,2812 "n_items": 272,2813 "n_groups": 522814 },2815 "rougeL": {2816 "mean": 0.6493307386480731,2817 "ci_low": 0.6357035897069415,2818 "ci_high": 0.6645981784549918,2819 "n_items": 272,2820 "n_groups": 522821 },2822 "exact_match": {2823 "mean": 0.0,2824 "ci_low": 0.0,2825 "ci_high": 0.0,2826 "n_items": 272,2827 "n_groups": 522828 },2829 "number_recall": {2830 "mean": 0.9890280304705614,2831 "ci_low": 0.9790548054194369,2832 "ci_high": 0.9958697730441934,2833 "n_items": 272,2834 "n_groups": 522835 },2836 "placeholders_exact": {2837 "mean": 0.9963235294117647,2838 "ci_low": 0.987600723140496,2839 "ci_high": 1.0,2840 "n_items": 272,2841 "n_groups": 522842 }2843 }2844 },2845 "by_discipline": {2846 "biomedicine": {2847 "n": 380,2848 "chrf": {2849 "mean": 70.59515435777637,2850 "ci_low": 69.41817790992866,2851 "ci_high": 71.65887252778539,2852 "n_items": 380,2853 "n_groups": 312854 },2855 "edit_sim": {2856 "mean": 0.656983952932853,2857 "ci_low": 0.6403744525112484,2858 "ci_high": 0.6685899718016136,2859 "n_items": 380,2860 "n_groups": 312861 },2862 "rougeL": {2863 "mean": 0.674287605020663,2864 "ci_low": 0.6591149731528327,2865 "ci_high": 0.6851364354954366,2866 "n_items": 380,2867 "n_groups": 312868 },2869 "exact_match": {2870 "mean": 0.0,2871 "ci_low": 0.0,2872 "ci_high": 0.0,2873 "n_items": 380,2874 "n_groups": 312875 },2876 "number_recall": {2877 "mean": 0.9913574806234087,2878 "ci_low": 0.9844606692575241,2879 "ci_high": 0.9971684776969592,2880 "n_items": 380,2881 "n_groups": 312882 },2883 "placeholders_exact": {2884 "mean": 0.9921052631578947,2885 "ci_low": 0.9784565217391304,2886 "ci_high": 1.0,2887 "n_items": 380,2888 "n_groups": 312889 }2890 },2891 "engineering": {2892 "n": 12,2893 "chrf": {2894 "mean": 74.4466180849069,2895 "ci_low": 74.20031920263439,2896 "ci_high": 74.6929169671794,2897 "n_items": 12,2898 "n_groups": 22899 },2900 "edit_sim": {2901 "mean": 0.7099664508337016,2902 "ci_low": 0.6990838957566026,2903 "ci_high": 0.7208490059108006,2904 "n_items": 12,2905 "n_groups": 22906 },2907 "rougeL": {2908 "mean": 0.7096011406878068,2909 "ci_low": 0.6846786087572222,2910 "ci_high": 0.7345236726183914,2911 "n_items": 12,2912 "n_groups": 22913 },2914 "exact_match": {2915 "mean": 0.0,2916 "ci_low": 0.0,2917 "ci_high": 0.0,2918 "n_items": 12,2919 "n_groups": 22920 },2921 "number_recall": {2922 "mean": 1.0,2923 "ci_low": 1.0,2924 "ci_high": 1.0,2925 "n_items": 12,2926 "n_groups": 22927 },2928 "placeholders_exact": {2929 "mean": 1.0,2930 "ci_low": 1.0,2931 "ci_high": 1.0,2932 "n_items": 12,2933 "n_groups": 22934 }2935 },2936 "finance": {2937 "n": 96,2938 "chrf": {2939 "mean": 68.29930832042929,2940 "ci_low": 68.29930832042929,2941 "ci_high": 68.29930832042929,2942 "n_items": 96,2943 "n_groups": 12944 },2945 "edit_sim": {2946 "mean": 0.6500659336057343,2947 "ci_low": 0.6500659336057343,2948 "ci_high": 0.6500659336057343,2949 "n_items": 96,2950 "n_groups": 12951 },2952 "rougeL": {2953 "mean": 0.6465904735557021,2954 "ci_low": 0.6465904735557021,2955 "ci_high": 0.6465904735557021,2956 "n_items": 96,2957 "n_groups": 12958 },2959 "exact_match": {2960 "mean": 0.0,2961 "ci_low": 0.0,2962 "ci_high": 0.0,2963 "n_items": 96,2964 "n_groups": 12965 },2966 "number_recall": {2967 "mean": 1.0,2968 "ci_low": 1.0,2969 "ci_high": 1.0,2970 "n_items": 96,2971 "n_groups": 12972 },2973 "placeholders_exact": {2974 "mean": 1.0,2975 "ci_low": 1.0,2976 "ci_high": 1.0,2977 "n_items": 96,2978 "n_groups": 12979 }2980 },2981 "life_sciences": {2982 "n": 162,2983 "chrf": {2984 "mean": 71.7113745064178,2985 "ci_low": 69.40042384246875,2986 "ci_high": 74.26423776005113,2987 "n_items": 162,2988 "n_groups": 152989 },2990 "edit_sim": {2991 "mean": 0.6788147367633551,2992 "ci_low": 0.6564418517003823,2993 "ci_high": 0.7047089196833342,2994 "n_items": 162,2995 "n_groups": 152996 },2997 "rougeL": {2998 "mean": 0.6960854987396047,2999 "ci_low": 0.6699903186110575,3000 "ci_high": 0.7258073233884126,3001 "n_items": 162,3002 "n_groups": 153003 },3004 "exact_match": {3005 "mean": 0.0,3006 "ci_low": 0.0,3007 "ci_high": 0.0,3008 "n_items": 162,3009 "n_groups": 153010 },3011 "number_recall": {3012 "mean": 0.9856179057567948,3013 "ci_low": 0.9705704581876456,3014 "ci_high": 0.997524513488055,3015 "n_items": 162,3016 "n_groups": 153017 },3018 "placeholders_exact": {3019 "mean": 0.9876543209876543,3020 "ci_low": 0.9704131054131053,3021 "ci_high": 1.0,3022 "n_items": 162,3023 "n_groups": 153024 }3025 },3026 "mathematics": {3027 "n": 60,3028 "chrf": {3029 "mean": 69.96312730474226,3030 "ci_low": 69.96312730474226,3031 "ci_high": 69.96312730474226,3032 "n_items": 60,3033 "n_groups": 13034 },3035 "edit_sim": {3036 "mean": 0.6992019630966972,3037 "ci_low": 0.6992019630966972,3038 "ci_high": 0.6992019630966972,3039 "n_items": 60,3040 "n_groups": 13041 },3042 "rougeL": {3043 "mean": 0.6896786327206967,3044 "ci_low": 0.6896786327206967,3045 "ci_high": 0.6896786327206967,3046 "n_items": 60,3047 "n_groups": 13048 },3049 "exact_match": {3050 "mean": 0.0,3051 "ci_low": 0.0,3052 "ci_high": 0.0,3053 "n_items": 60,3054 "n_groups": 13055 },3056 "number_recall": {3057 "mean": 1.0,3058 "ci_low": 1.0,3059 "ci_high": 1.0,3060 "n_items": 60,3061 "n_groups": 13062 },3063 "placeholders_exact": {3064 "mean": 1.0,3065 "ci_low": 1.0,3066 "ci_high": 1.0,3067 "n_items": 60,3068 "n_groups": 13069 }3070 },3071 "social_sciences": {3072 "n": 30,3073 "chrf": {3074 "mean": 68.34130593546618,3075 "ci_low": 56.75584666192979,3076 "ci_high": 71.23767075385028,3077 "n_items": 30,3078 "n_groups": 23079 },3080 "edit_sim": {3081 "mean": 0.6831564598432772,3082 "ci_low": 0.5774919170503715,3083 "ci_high": 0.7095725955415038,3084 "n_items": 30,3085 "n_groups": 23086 },3087 "rougeL": {3088 "mean": 0.6748262950819864,3089 "ci_low": 0.487761131591071,3090 "ci_high": 0.7215925859547153,3091 "n_items": 30,3092 "n_groups": 23093 },3094 "exact_match": {3095 "mean": 0.0,3096 "ci_low": 0.0,3097 "ci_high": 0.0,3098 "n_items": 30,3099 "n_groups": 23100 },3101 "number_recall": {3102 "mean": 0.9277777777777777,3103 "ci_low": 0.9097222222222223,3104 "ci_high": 1.0,3105 "n_items": 30,3106 "n_groups": 23107 },3108 "placeholders_exact": {3109 "mean": 1.0,3110 "ci_low": 1.0,3111 "ci_high": 1.0,3112 "n_items": 30,3113 "n_groups": 23114 }3115 },3116 "statistics": {3117 "n": 72,3118 "chrf": {3119 "mean": 68.33972206171237,3120 "ci_low": 65.27554363547439,3121 "ci_high": 69.87181127483133,3122 "n_items": 72,3123 "n_groups": 23124 },3125 "edit_sim": {3126 "mean": 0.6249694995392675,3127 "ci_low": 0.6023731247172015,3128 "ci_high": 0.6362676869503004,3129 "n_items": 72,3130 "n_groups": 23131 },3132 "rougeL": {3133 "mean": 0.6535806213396766,3134 "ci_low": 0.6412993856594137,3135 "ci_high": 0.6597212391798083,3136 "n_items": 72,3137 "n_groups": 23138 },3139 "exact_match": {3140 "mean": 0.0,3141 "ci_low": 0.0,3142 "ci_high": 0.0,3143 "n_items": 72,3144 "n_groups": 23145 },3146 "number_recall": {3147 "mean": 0.9953703703703705,3148 "ci_low": 0.9930555555555555,3149 "ci_high": 1.0,3150 "n_items": 72,3151 "n_groups": 23152 },3153 "placeholders_exact": {3154 "mean": 1.0,3155 "ci_low": 1.0,3156 "ci_high": 1.0,3157 "n_items": 72,3158 "n_groups": 23159 }3160 }3161 },3162 "by_era": {3163 "1980-1999": {3164 "n": 8,3165 "chrf": {3166 "mean": 69.93694505304498,3167 "ci_low": 69.54794035702074,3168 "ci_high": 70.58528621308538,3169 "n_items": 8,3170 "n_groups": 23171 },3172 "edit_sim": {3173 "mean": 0.6842131737063628,3174 "ci_low": 0.6694544689391505,3175 "ci_high": 0.7088110149850504,3176 "n_items": 8,3177 "n_groups": 23178 },3179 "rougeL": {3180 "mean": 0.6823286772236912,3181 "ci_low": 0.6610513453424772,3182 "ci_high": 0.7177908970257144,3183 "n_items": 8,3184 "n_groups": 23185 },3186 "exact_match": {3187 "mean": 0.0,3188 "ci_low": 0.0,3189 "ci_high": 0.0,3190 "n_items": 8,3191 "n_groups": 23192 },3193 "number_recall": {3194 "mean": 1.0,3195 "ci_low": 1.0,3196 "ci_high": 1.0,3197 "n_items": 8,3198 "n_groups": 23199 },3200 "placeholders_exact": {3201 "mean": 1.0,3202 "ci_low": 1.0,3203 "ci_high": 1.0,3204 "n_items": 8,3205 "n_groups": 23206 }3207 },3208 "2000-2009": {3209 "n": 89,3210 "chrf": {3211 "mean": 68.46448022374248,3212 "ci_low": 66.1587584545427,3213 "ci_high": 72.63113521524679,3214 "n_items": 89,3215 "n_groups": 73216 },3217 "edit_sim": {3218 "mean": 0.6485558671732253,3219 "ci_low": 0.619080024454127,3220 "ci_high": 0.7001487307464134,3221 "n_items": 89,3222 "n_groups": 73223 },3224 "rougeL": {3225 "mean": 0.6606854729824952,3226 "ci_low": 0.6339121285713069,3227 "ci_high": 0.7135605884507036,3228 "n_items": 89,3229 "n_groups": 73230 },3231 "exact_match": {3232 "mean": 0.0,3233 "ci_low": 0.0,3234 "ci_high": 0.0,3235 "n_items": 89,3236 "n_groups": 73237 },3238 "number_recall": {3239 "mean": 0.9948501872659177,3240 "ci_low": 0.9859222304932049,3241 "ci_high": 1.0,3242 "n_items": 89,3243 "n_groups": 73244 },3245 "placeholders_exact": {3246 "mean": 0.9887640449438202,3247 "ci_low": 0.9658992946708465,3248 "ci_high": 1.0,3249 "n_items": 89,3250 "n_groups": 73251 }3252 },3253 "2010-2017": {3254 "n": 715,3255 "chrf": {3256 "mean": 70.5023032814394,3257 "ci_low": 69.59401281194774,3258 "ci_high": 71.61013845623563,3259 "n_items": 715,3260 "n_groups": 453261 },3262 "edit_sim": {3263 "mean": 0.6640521041764504,3264 "ci_low": 0.651794447704775,3265 "ci_high": 0.6760671158770307,3266 "n_items": 715,3267 "n_groups": 453268 },3269 "rougeL": {3270 "mean": 0.676932468380693,3271 "ci_low": 0.6641248925967156,3272 "ci_high": 0.6899165201946655,3273 "n_items": 715,3274 "n_groups": 453275 },3276 "exact_match": {3277 "mean": 0.0,3278 "ci_low": 0.0,3279 "ci_high": 0.0,3280 "n_items": 715,3281 "n_groups": 453282 },3283 "number_recall": {3284 "mean": 0.9892926946892718,3285 "ci_low": 0.9800398418551043,3286 "ci_high": 0.9956865352036989,3287 "n_items": 715,3288 "n_groups": 453289 },3290 "placeholders_exact": {3291 "mean": 0.9944055944055944,3292 "ci_low": 0.9864854689026376,3293 "ci_high": 1.0,3294 "n_items": 715,3295 "n_groups": 453296 }3297 }3298 },3299 "by_role": {3300 "abstract": {3301 "n": 83,3302 "chrf": {3303 "mean": 72.74143723899563,3304 "ci_low": 70.93201339720855,3305 "ci_high": 74.47887519919658,3306 "n_items": 83,3307 "n_groups": 153308 },3309 "edit_sim": {3310 "mean": 0.6887033548129731,3311 "ci_low": 0.6638561142643623,3312 "ci_high": 0.7140649572229821,3313 "n_items": 83,3314 "n_groups": 153315 },3316 "rougeL": {3317 "mean": 0.7009677152616665,3318 "ci_low": 0.6730565623283377,3319 "ci_high": 0.7304102588340222,3320 "n_items": 83,3321 "n_groups": 153322 },3323 "exact_match": {3324 "mean": 0.0,3325 "ci_low": 0.0,3326 "ci_high": 0.0,3327 "n_items": 83,3328 "n_groups": 153329 },3330 "number_recall": {3331 "mean": 0.9928806133625411,3332 "ci_low": 0.979230324074074,3333 "ci_high": 1.0,3334 "n_items": 83,3335 "n_groups": 153336 },3337 "placeholders_exact": {3338 "mean": 1.0,3339 "ci_low": 1.0,3340 "ci_high": 1.0,3341 "n_items": 83,3342 "n_groups": 153343 }3344 },3345 "background": {3346 "n": 58,3347 "chrf": {3348 "mean": 70.3479758009031,3349 "ci_low": 67.31164977787914,3350 "ci_high": 73.3207574552499,3351 "n_items": 58,3352 "n_groups": 73353 },3354 "edit_sim": {3355 "mean": 0.6722393176730548,3356 "ci_low": 0.6386211499200647,3357 "ci_high": 0.7099595375242705,3358 "n_items": 58,3359 "n_groups": 73360 },3361 "rougeL": {3362 "mean": 0.6798982558981467,3363 "ci_low": 0.6531488578911425,3364 "ci_high": 0.708573739466385,3365 "n_items": 58,3366 "n_groups": 73367 },3368 "exact_match": {3369 "mean": 0.0,3370 "ci_low": 0.0,3371 "ci_high": 0.0,3372 "n_items": 58,3373 "n_groups": 73374 },3375 "number_recall": {3376 "mean": 0.9923371647509579,3377 "ci_low": 0.9824561403508771,3378 "ci_high": 1.0,3379 "n_items": 58,3380 "n_groups": 73381 },3382 "placeholders_exact": {3383 "mean": 0.9827586206896551,3384 "ci_low": 0.9516129032258065,3385 "ci_high": 1.0,3386 "n_items": 58,3387 "n_groups": 73388 }3389 },3390 "conclusion": {3391 "n": 47,3392 "chrf": {3393 "mean": 69.45827126241116,3394 "ci_low": 67.2838954350584,3395 "ci_high": 71.34150972139884,3396 "n_items": 47,3397 "n_groups": 83398 },3399 "edit_sim": {3400 "mean": 0.6731232085408914,3401 "ci_low": 0.6482698643675356,3402 "ci_high": 0.6983573251587756,3403 "n_items": 47,3404 "n_groups": 83405 },3406 "rougeL": {3407 "mean": 0.6727611972240978,3408 "ci_low": 0.6488622838717074,3409 "ci_high": 0.6925846491969183,3410 "n_items": 47,3411 "n_groups": 83412 },3413 "exact_match": {3414 "mean": 0.0,3415 "ci_low": 0.0,3416 "ci_high": 0.0,3417 "n_items": 47,3418 "n_groups": 83419 },3420 "number_recall": {3421 "mean": 0.9787234042553191,3422 "ci_low": 0.9361702127659575,3423 "ci_high": 1.0,3424 "n_items": 47,3425 "n_groups": 83426 },3427 "placeholders_exact": {3428 "mean": 1.0,3429 "ci_low": 1.0,3430 "ci_high": 1.0,3431 "n_items": 47,3432 "n_groups": 83433 }3434 },3435 "data": {3436 "n": 69,3437 "chrf": {3438 "mean": 72.1196739908267,3439 "ci_low": 69.58959062488948,3440 "ci_high": 74.88965043364232,3441 "n_items": 69,3442 "n_groups": 103443 },3444 "edit_sim": {3445 "mean": 0.6404832247196582,3446 "ci_low": 0.5904396789608839,3447 "ci_high": 0.6887241690976067,3448 "n_items": 69,3449 "n_groups": 103450 },3451 "rougeL": {3452 "mean": 0.6764049869395794,3453 "ci_low": 0.6366335480011656,3454 "ci_high": 0.7198799116268972,3455 "n_items": 69,3456 "n_groups": 103457 },3458 "exact_match": {3459 "mean": 0.0,3460 "ci_low": 0.0,3461 "ci_high": 0.0,3462 "n_items": 69,3463 "n_groups": 103464 },3465 "number_recall": {3466 "mean": 0.9868421052631579,3467 "ci_low": 0.9605263157894737,3468 "ci_high": 1.0,3469 "n_items": 69,3470 "n_groups": 103471 },3472 "placeholders_exact": {3473 "mean": 0.9710144927536232,3474 "ci_low": 0.9015983606557377,3475 "ci_high": 1.0,3476 "n_items": 69,3477 "n_groups": 103478 }3479 },3480 "discussion": {3481 "n": 103,3482 "chrf": {3483 "mean": 70.47269147709424,3484 "ci_low": 68.68148842746908,3485 "ci_high": 72.54798609598821,3486 "n_items": 103,3487 "n_groups": 143488 },3489 "edit_sim": {3490 "mean": 0.6806859697168846,3491 "ci_low": 0.6588783112074971,3492 "ci_high": 0.7070494733893578,3493 "n_items": 103,3494 "n_groups": 143495 },3496 "rougeL": {3497 "mean": 0.6901010620736194,3498 "ci_low": 0.6620154442539147,3499 "ci_high": 0.7196051476409525,3500 "n_items": 103,3501 "n_groups": 143502 },3503 "exact_match": {3504 "mean": 0.0,3505 "ci_low": 0.0,3506 "ci_high": 0.0,3507 "n_items": 103,3508 "n_groups": 143509 },3510 "number_recall": {3511 "mean": 0.9838187702265372,3512 "ci_low": 0.95,3513 "ci_high": 1.0,3514 "n_items": 103,3515 "n_groups": 143516 },3517 "placeholders_exact": {3518 "mean": 1.0,3519 "ci_low": 1.0,3520 "ci_high": 1.0,3521 "n_items": 103,3522 "n_groups": 143523 }3524 },3525 "introduction": {3526 "n": 99,3527 "chrf": {3528 "mean": 69.81277814918845,3529 "ci_low": 67.19677906172261,3530 "ci_high": 72.08137273505234,3531 "n_items": 99,3532 "n_groups": 143533 },3534 "edit_sim": {3535 "mean": 0.6519512369587331,3536 "ci_low": 0.617000580323305,3537 "ci_high": 0.6849365604565273,3538 "n_items": 99,3539 "n_groups": 143540 },3541 "rougeL": {3542 "mean": 0.6620403673863906,3543 "ci_low": 0.6343176047865587,3544 "ci_high": 0.6894765292889352,3545 "n_items": 99,3546 "n_groups": 143547 },3548 "exact_match": {3549 "mean": 0.0,3550 "ci_low": 0.0,3551 "ci_high": 0.0,3552 "n_items": 99,3553 "n_groups": 143554 },3555 "number_recall": {3556 "mean": 0.9952020202020203,3557 "ci_low": 0.9848765432098766,3558 "ci_high": 1.0,3559 "n_items": 99,3560 "n_groups": 143561 },3562 "placeholders_exact": {3563 "mean": 1.0,3564 "ci_low": 1.0,3565 "ci_high": 1.0,3566 "n_items": 99,3567 "n_groups": 143568 }3569 },3570 "limitations": {3571 "n": 42,3572 "chrf": {3573 "mean": 70.1372404279545,3574 "ci_low": 68.57182595802757,3575 "ci_high": 71.76345317025455,3576 "n_items": 42,3577 "n_groups": 43578 },3579 "edit_sim": {3580 "mean": 0.6600396676697666,3581 "ci_low": 0.617757771841755,3582 "ci_high": 0.6898774774123402,3583 "n_items": 42,3584 "n_groups": 43585 },3586 "rougeL": {3587 "mean": 0.6623571386527315,3588 "ci_low": 0.6396848457832429,3589 "ci_high": 0.6815531854546396,3590 "n_items": 42,3591 "n_groups": 43592 },3593 "exact_match": {3594 "mean": 0.0,3595 "ci_low": 0.0,3596 "ci_high": 0.0,3597 "n_items": 42,3598 "n_groups": 43599 },3600 "number_recall": {3601 "mean": 1.0,3602 "ci_low": 1.0,3603 "ci_high": 1.0,3604 "n_items": 42,3605 "n_groups": 43606 },3607 "placeholders_exact": {3608 "mean": 1.0,3609 "ci_low": 1.0,3610 "ci_high": 1.0,3611 "n_items": 42,3612 "n_groups": 43613 }3614 },3615 "methods": {3616 "n": 98,3617 "chrf": {3618 "mean": 69.56763637679882,3619 "ci_low": 66.3353227069222,3620 "ci_high": 73.25551495393748,3621 "n_items": 98,3622 "n_groups": 123623 },3624 "edit_sim": {3625 "mean": 0.658740659014652,3626 "ci_low": 0.6372568372712644,3627 "ci_high": 0.6819657827235521,3628 "n_items": 98,3629 "n_groups": 123630 },3631 "rougeL": {3632 "mean": 0.670627292687944,3633 "ci_low": 0.6433344759011197,3634 "ci_high": 0.699756288315857,3635 "n_items": 98,3636 "n_groups": 123637 },3638 "exact_match": {3639 "mean": 0.0,3640 "ci_low": 0.0,3641 "ci_high": 0.0,3642 "n_items": 98,3643 "n_groups": 123644 },3645 "number_recall": {3646 "mean": 0.9930426716141001,3647 "ci_low": 0.979978354978355,3648 "ci_high": 1.0,3649 "n_items": 98,3650 "n_groups": 123651 },3652 "placeholders_exact": {3653 "mean": 1.0,3654 "ci_low": 1.0,3655 "ci_high": 1.0,3656 "n_items": 98,3657 "n_groups": 123658 }3659 },3660 "other": {3661 "n": 70,3662 "chrf": {3663 "mean": 68.59654874949047,3664 "ci_low": 64.11745614032036,3665 "ci_high": 72.36988961718158,3666 "n_items": 70,3667 "n_groups": 93668 },3669 "edit_sim": {3670 "mean": 0.6555521082386644,3671 "ci_low": 0.6181835588682577,3672 "ci_high": 0.6883420177993023,3673 "n_items": 70,3674 "n_groups": 93675 },3676 "rougeL": {3677 "mean": 0.6560397506156181,3678 "ci_low": 0.6020798337949927,3679 "ci_high": 0.7031732294878662,3680 "n_items": 70,3681 "n_groups": 93682 },3683 "exact_match": {3684 "mean": 0.0,3685 "ci_low": 0.0,3686 "ci_high": 0.0,3687 "n_items": 70,3688 "n_groups": 93689 },3690 "number_recall": {3691 "mean": 0.989702380952381,3692 "ci_low": 0.9790190034918077,3693 "ci_high": 0.9973591549295775,3694 "n_items": 70,3695 "n_groups": 93696 },3697 "placeholders_exact": {3698 "mean": 0.9714285714285714,3699 "ci_low": 0.941843365253078,3700 "ci_high": 1.0,3701 "n_items": 70,3702 "n_groups": 93703 }3704 },3705 "related_work": {3706 "n": 6,3707 "chrf": {3708 "mean": 74.6929169671794,3709 "ci_low": 74.6929169671794,3710 "ci_high": 74.6929169671794,3711 "n_items": 6,3712 "n_groups": 13713 },3714 "edit_sim": {3715 "mean": 0.7208490059108007,3716 "ci_low": 0.7208490059108007,3717 "ci_high": 0.7208490059108007,3718 "n_items": 6,3719 "n_groups": 13720 },3721 "rougeL": {3722 "mean": 0.7345236726183915,3723 "ci_low": 0.7345236726183915,3724 "ci_high": 0.7345236726183915,3725 "n_items": 6,3726 "n_groups": 13727 },3728 "exact_match": {3729 "mean": 0.0,3730 "ci_low": 0.0,3731 "ci_high": 0.0,3732 "n_items": 6,3733 "n_groups": 13734 },3735 "number_recall": {3736 "mean": 1.0,3737 "ci_low": 1.0,3738 "ci_high": 1.0,3739 "n_items": 6,3740 "n_groups": 13741 },3742 "placeholders_exact": {3743 "mean": 1.0,3744 "ci_low": 1.0,3745 "ci_high": 1.0,3746 "n_items": 6,3747 "n_groups": 13748 }3749 },3750 "results": {3751 "n": 78,3752 "chrf": {3753 "mean": 68.44605765774541,3754 "ci_low": 67.44432445657318,3755 "ci_high": 69.5565754311798,3756 "n_items": 78,3757 "n_groups": 93758 },3759 "edit_sim": {3760 "mean": 0.6257706845251485,3761 "ci_low": 0.5945315432052294,3762 "ci_high": 0.6549800694670721,3763 "n_items": 78,3764 "n_groups": 93765 },3766 "rougeL": {3767 "mean": 0.6580059286690347,3768 "ci_low": 0.6339882656533994,3769 "ci_high": 0.6787336476499006,3770 "n_items": 78,3771 "n_groups": 93772 },3773 "exact_match": {3774 "mean": 0.0,3775 "ci_low": 0.0,3776 "ci_high": 0.0,3777 "n_items": 78,3778 "n_groups": 93779 },3780 "number_recall": {3781 "mean": 0.9917887667887667,3782 "ci_low": 0.9841697099892588,3783 "ci_high": 1.0,3784 "n_items": 78,3785 "n_groups": 93786 },3787 "placeholders_exact": {3788 "mean": 1.0,3789 "ci_low": 1.0,3790 "ci_high": 1.0,3791 "n_items": 78,3792 "n_groups": 93793 }3794 },3795 "theory": {3796 "n": 59,3797 "chrf": {3798 "mean": 70.86796391855826,3799 "ci_low": 69.47503608726586,3800 "ci_high": 72.77132315573978,3801 "n_items": 59,3802 "n_groups": 63803 },3804 "edit_sim": {3805 "mean": 0.6788831678336448,3806 "ci_low": 0.6633894400105257,3807 "ci_high": 0.6917549577659512,3808 "n_items": 59,3809 "n_groups": 63810 },3811 "rougeL": {3812 "mean": 0.6871685515524688,3813 "ci_low": 0.6736314803265948,3814 "ci_high": 0.6993899656055993,3815 "n_items": 59,3816 "n_groups": 63817 },3818 "exact_match": {3819 "mean": 0.0,3820 "ci_low": 0.0,3821 "ci_high": 0.0,3822 "n_items": 59,3823 "n_groups": 63824 },3825 "number_recall": {3826 "mean": 0.9832878985421358,3827 "ci_low": 0.9481045270518955,3828 "ci_high": 1.0,3829 "n_items": 59,3830 "n_groups": 63831 },3832 "placeholders_exact": {3833 "mean": 1.0,3834 "ci_low": 1.0,3835 "ci_high": 1.0,3836 "n_items": 59,3837 "n_groups": 63838 }3839 }3840 }3841 },3842 "htr-m1-tiny-s3": {3843 "overall": {3844 "n": 812,3845 "chrf": {3846 "mean": 70.61817272526466,3847 "ci_low": 69.78529149974108,3848 "ci_high": 71.58042672849739,3849 "n_items": 812,3850 "n_groups": 543851 },3852 "edit_sim": {3853 "mean": 0.6689694497349163,3854 "ci_low": 0.6574550093144585,3855 "ci_high": 0.6808092216580378,3856 "n_items": 812,3857 "n_groups": 543858 },3859 "rougeL": {3860 "mean": 0.6801033689438778,3861 "ci_low": 0.6686675605083847,3862 "ci_high": 0.6924514643721377,3863 "n_items": 812,3864 "n_groups": 543865 },3866 "exact_match": {3867 "mean": 0.0,3868 "ci_low": 0.0,3869 "ci_high": 0.0,3870 "n_items": 812,3871 "n_groups": 543872 },3873 "number_recall": {3874 "mean": 0.9889100230136009,3875 "ci_low": 0.9816287880265608,3876 "ci_high": 0.9950339087631813,3877 "n_items": 812,3878 "n_groups": 543879 },3880 "placeholders_exact": {3881 "mean": 0.9950738916256158,3882 "ci_low": 0.9848484848484849,3883 "ci_high": 1.0,3884 "n_items": 812,3885 "n_groups": 543886 }3887 },3888 "by_variant": {3889 "A": {3890 "n": 133,3891 "chrf": {3892 "mean": 69.30537419344762,3893 "ci_low": 68.40450903069164,3894 "ci_high": 70.22108543822505,3895 "n_items": 133,3896 "n_groups": 503897 },3898 "edit_sim": {3899 "mean": 0.6426677412906483,3900 "ci_low": 0.6287082765107835,3901 "ci_high": 0.6547304439976586,3902 "n_items": 133,3903 "n_groups": 503904 },3905 "rougeL": {3906 "mean": 0.6535881743479635,3907 "ci_low": 0.6400671388754519,3908 "ci_high": 0.6668940377985457,3909 "n_items": 133,3910 "n_groups": 503911 },3912 "exact_match": {3913 "mean": 0.0,3914 "ci_low": 0.0,3915 "ci_high": 0.0,3916 "n_items": 133,3917 "n_groups": 503918 },3919 "number_recall": {3920 "mean": 0.9790756796296961,3921 "ci_low": 0.9600334259117003,3922 "ci_high": 0.9964990226570768,3923 "n_items": 133,3924 "n_groups": 503925 },3926 "placeholders_exact": {3927 "mean": 0.9924812030075187,3928 "ci_low": 0.9735084686163042,3929 "ci_high": 1.0,3930 "n_items": 133,3931 "n_groups": 503932 }3933 },3934 "B": {3935 "n": 136,3936 "chrf": {3937 "mean": 75.5638145361216,3938 "ci_low": 74.42211078219982,3939 "ci_high": 76.80655100212103,3940 "n_items": 136,3941 "n_groups": 523942 },3943 "edit_sim": {3944 "mean": 0.7449933834370868,3945 "ci_low": 0.7305380885824625,3946 "ci_high": 0.758073762857265,3947 "n_items": 136,3948 "n_groups": 523949 },3950 "rougeL": {3951 "mean": 0.7481738204587757,3952 "ci_low": 0.7345634560503289,3953 "ci_high": 0.7627743640880345,3954 "n_items": 136,3955 "n_groups": 523956 },3957 "exact_match": {3958 "mean": 0.0,3959 "ci_low": 0.0,3960 "ci_high": 0.0,3961 "n_items": 136,3962 "n_groups": 523963 },3964 "number_recall": {3965 "mean": 0.9982092597804673,3966 "ci_low": 0.9956884748726854,3967 "ci_high": 1.0,3968 "n_items": 136,3969 "n_groups": 523970 },3971 "placeholders_exact": {3972 "mean": 0.9926470588235294,3973 "ci_low": 0.9734513274336283,3974 "ci_high": 1.0,3975 "n_items": 136,3976 "n_groups": 523977 }3978 },3979 "C": {3980 "n": 135,3981 "chrf": {3982 "mean": 72.28077404682723,3983 "ci_low": 71.13126258366646,3984 "ci_high": 73.66041282897116,3985 "n_items": 135,3986 "n_groups": 533987 },3988 "edit_sim": {3989 "mean": 0.7167711829380455,3990 "ci_low": 0.7035821068218058,3991 "ci_high": 0.7314327165457993,3992 "n_items": 135,3993 "n_groups": 533994 },3995 "rougeL": {3996 "mean": 0.7352838711069374,3997 "ci_low": 0.7212852772690035,3998 "ci_high": 0.7511228650962566,3999 "n_items": 135,4000 "n_groups": 534001 },4002 "exact_match": {4003 "mean": 0.0,4004 "ci_low": 0.0,4005 "ci_high": 0.0,4006 "n_items": 135,4007 "n_groups": 534008 },4009 "number_recall": {4010 "mean": 0.9942311327399047,4011 "ci_low": 0.9864754328020074,4012 "ci_high": 0.9989546721040615,4013 "n_items": 135,4014 "n_groups": 534015 },4016 "placeholders_exact": {4017 "mean": 1.0,4018 "ci_low": 1.0,4019 "ci_high": 1.0,4020 "n_items": 135,4021 "n_groups": 534022 }4023 },4024 "D": {4025 "n": 136,4026 "chrf": {4027 "mean": 68.13068527097288,4028 "ci_low": 67.07971612147182,4029 "ci_high": 69.17854666755552,4030 "n_items": 136,4031 "n_groups": 514032 },4033 "edit_sim": {4034 "mean": 0.6183640238403697,4035 "ci_low": 0.6034439924778181,4036 "ci_high": 0.6344975408752931,4037 "n_items": 136,4038 "n_groups": 514039 },4040 "rougeL": {4041 "mean": 0.6328960173352148,4042 "ci_low": 0.6187861868817349,4043 "ci_high": 0.647344012098133,4044 "n_items": 136,4045 "n_groups": 514046 },4047 "exact_match": {4048 "mean": 0.0,4049 "ci_low": 0.0,4050 "ci_high": 0.0,4051 "n_items": 136,4052 "n_groups": 514053 },4054 "number_recall": {4055 "mean": 0.9781549880717836,4056 "ci_low": 0.9546500441013,4057 "ci_high": 0.9970180080696659,4058 "n_items": 136,4059 "n_groups": 514060 },4061 "placeholders_exact": {4062 "mean": 1.0,4063 "ci_low": 1.0,4064 "ci_high": 1.0,4065 "n_items": 136,4066 "n_groups": 514067 }4068 },4069 "E": {4070 "n": 272,4071 "chrf": {4072 "mean": 69.20582726139644,4073 "ci_low": 68.27203573517836,4074 "ci_high": 70.24912805432635,4075 "n_items": 272,4076 "n_groups": 524077 },4078 "edit_sim": {4079 "mean": 0.645395832745315,4080 "ci_low": 0.6307406394380783,4081 "ci_high": 0.6619331107068049,4082 "n_items": 272,4083 "n_groups": 524084 },4085 "rougeL": {4086 "mean": 0.6552495877012147,4087 "ci_low": 0.641488323298433,4088 "ci_high": 0.6706515280809408,4089 "n_items": 272,4090 "n_groups": 524091 },4092 "exact_match": {4093 "mean": 0.0,4094 "ci_low": 0.0,4095 "ci_high": 0.0,4096 "n_items": 272,4097 "n_groups": 524098 },4099 "number_recall": {4100 "mean": 0.9918056348106657,4101 "ci_low": 0.9843340856668892,4102 "ci_high": 0.996935560033276,4103 "n_items": 272,4104 "n_groups": 524105 },4106 "placeholders_exact": {4107 "mean": 0.9926470588235294,4108 "ci_low": 0.9801970375608989,4109 "ci_high": 1.0,4110 "n_items": 272,4111 "n_groups": 524112 }4113 }4114 },4115 "by_discipline": {4116 "biomedicine": {4117 "n": 380,4118 "chrf": {4119 "mean": 71.07440778809145,4120 "ci_low": 69.84917417695321,4121 "ci_high": 72.17051800397317,4122 "n_items": 380,4123 "n_groups": 314124 },4125 "edit_sim": {4126 "mean": 0.6668789963071383,4127 "ci_low": 0.6497557592098059,4128 "ci_high": 0.6786619381190239,4129 "n_items": 380,4130 "n_groups": 314131 },4132 "rougeL": {4133 "mean": 0.6823994817665048,4134 "ci_low": 0.666475358131465,4135 "ci_high": 0.693599151524129,4136 "n_items": 380,4137 "n_groups": 314138 },4139 "exact_match": {4140 "mean": 0.0,4141 "ci_low": 0.0,4142 "ci_high": 0.0,4143 "n_items": 380,4144 "n_groups": 314145 },4146 "number_recall": {4147 "mean": 0.9890407188225747,4148 "ci_low": 0.9792808855649323,4149 "ci_high": 0.9966070563502771,4150 "n_items": 380,4151 "n_groups": 314152 },4153 "placeholders_exact": {4154 "mean": 1.0,4155 "ci_low": 1.0,4156 "ci_high": 1.0,4157 "n_items": 380,4158 "n_groups": 314159 }4160 },4161 "engineering": {4162 "n": 12,4163 "chrf": {4164 "mean": 72.63960661360021,4165 "ci_low": 71.92439039936896,4166 "ci_high": 73.35482282783146,4167 "n_items": 12,4168 "n_groups": 24169 },4170 "edit_sim": {4171 "mean": 0.6995632530377011,4172 "ci_low": 0.6905803536188588,4173 "ci_high": 0.7085461524565435,4174 "n_items": 12,4175 "n_groups": 24176 },4177 "rougeL": {4178 "mean": 0.6893584663319491,4179 "ci_low": 0.6578878586071616,4180 "ci_high": 0.7208290740567366,4181 "n_items": 12,4182 "n_groups": 24183 },4184 "exact_match": {4185 "mean": 0.0,4186 "ci_low": 0.0,4187 "ci_high": 0.0,4188 "n_items": 12,4189 "n_groups": 24190 },4191 "number_recall": {4192 "mean": 1.0,4193 "ci_low": 1.0,4194 "ci_high": 1.0,4195 "n_items": 12,4196 "n_groups": 24197 },4198 "placeholders_exact": {4199 "mean": 1.0,4200 "ci_low": 1.0,4201 "ci_high": 1.0,4202 "n_items": 12,4203 "n_groups": 24204 }4205 },4206 "finance": {4207 "n": 96,4208 "chrf": {4209 "mean": 69.09118197907776,4210 "ci_low": 69.09118197907776,4211 "ci_high": 69.09118197907776,4212 "n_items": 96,4213 "n_groups": 14214 },4215 "edit_sim": {4216 "mean": 0.6553684518874499,4217 "ci_low": 0.6553684518874499,4218 "ci_high": 0.6553684518874499,4219 "n_items": 96,4220 "n_groups": 14221 },4222 "rougeL": {4223 "mean": 0.6542852021836064,4224 "ci_low": 0.6542852021836064,4225 "ci_high": 0.6542852021836064,4226 "n_items": 96,4227 "n_groups": 14228 },4229 "exact_match": {4230 "mean": 0.0,4231 "ci_low": 0.0,4232 "ci_high": 0.0,4233 "n_items": 96,4234 "n_groups": 14235 },4236 "number_recall": {4237 "mean": 0.9982638888888888,4238 "ci_low": 0.9982638888888888,4239 "ci_high": 0.9982638888888888,4240 "n_items": 96,4241 "n_groups": 14242 },4243 "placeholders_exact": {4244 "mean": 1.0,4245 "ci_low": 1.0,4246 "ci_high": 1.0,4247 "n_items": 96,4248 "n_groups": 14249 }4250 },4251 "life_sciences": {4252 "n": 162,4253 "chrf": {4254 "mean": 71.7154395757866,4255 "ci_low": 69.58573191537968,4256 "ci_high": 74.19592535804911,4257 "n_items": 162,4258 "n_groups": 154259 },4260 "edit_sim": {4261 "mean": 0.679253235414821,4262 "ci_low": 0.6578565894956341,4263 "ci_high": 0.7060457624907754,4264 "n_items": 162,4265 "n_groups": 154266 },4267 "rougeL": {4268 "mean": 0.6945656741908964,4269 "ci_low": 0.6702769669459208,4270 "ci_high": 0.7245865538109563,4271 "n_items": 162,4272 "n_groups": 154273 },4274 "exact_match": {4275 "mean": 0.0,4276 "ci_low": 0.0,4277 "ci_high": 0.0,4278 "n_items": 162,4279 "n_groups": 154280 },4281 "number_recall": {4282 "mean": 0.9865810629699518,4283 "ci_low": 0.9687662337662338,4284 "ci_high": 1.0,4285 "n_items": 162,4286 "n_groups": 154287 },4288 "placeholders_exact": {4289 "mean": 0.9814814814814815,4290 "ci_low": 0.9379310344827586,4291 "ci_high": 1.0,4292 "n_items": 162,4293 "n_groups": 154294 }4295 },4296 "mathematics": {4297 "n": 60,4298 "chrf": {4299 "mean": 70.18363583675207,4300 "ci_low": 70.18363583675207,4301 "ci_high": 70.18363583675207,4302 "n_items": 60,4303 "n_groups": 14304 },4305 "edit_sim": {4306 "mean": 0.7021554033278129,4307 "ci_low": 0.7021554033278129,4308 "ci_high": 0.7021554033278129,4309 "n_items": 60,4310 "n_groups": 14311 },4312 "rougeL": {4313 "mean": 0.6924922982972391,4314 "ci_low": 0.6924922982972391,4315 "ci_high": 0.6924922982972391,4316 "n_items": 60,4317 "n_groups": 14318 },4319 "exact_match": {4320 "mean": 0.0,4321 "ci_low": 0.0,4322 "ci_high": 0.0,4323 "n_items": 60,4324 "n_groups": 14325 },4326 "number_recall": {4327 "mean": 0.9666666666666667,4328 "ci_low": 0.9666666666666667,4329 "ci_high": 0.9666666666666667,4330 "n_items": 60,4331 "n_groups": 14332 },4333 "placeholders_exact": {4334 "mean": 1.0,4335 "ci_low": 1.0,4336 "ci_high": 1.0,4337 "n_items": 60,4338 "n_groups": 14339 }4340 },4341 "social_sciences": {4342 "n": 30,4343 "chrf": {4344 "mean": 68.84665526222638,4345 "ci_low": 56.72616650003939,4346 "ci_high": 71.87677745277311,4347 "n_items": 30,4348 "n_groups": 24349 },4350 "edit_sim": {4351 "mean": 0.6972837783875455,4352 "ci_low": 0.585016539019534,4353 "ci_high": 0.7253505882295487,4354 "n_items": 30,4355 "n_groups": 24356 },4357 "rougeL": {4358 "mean": 0.6828368654285991,4359 "ci_low": 0.48156473793247695,4360 "ci_high": 0.7331548973026295,4361 "n_items": 30,4362 "n_groups": 24363 },4364 "exact_match": {4365 "mean": 0.0,4366 "ci_low": 0.0,4367 "ci_high": 0.0,4368 "n_items": 30,4369 "n_groups": 24370 },4371 "number_recall": {4372 "mean": 1.0,4373 "ci_low": 1.0,4374 "ci_high": 1.0,4375 "n_items": 30,4376 "n_groups": 24377 },4378 "placeholders_exact": {4379 "mean": 1.0,4380 "ci_low": 1.0,4381 "ci_high": 1.0,4382 "n_items": 30,4383 "n_groups": 24384 }4385 },4386 "statistics": {4387 "n": 72,4388 "chrf": {4389 "mean": 68.54074339911287,4390 "ci_low": 66.37111463158372,4391 "ci_high": 69.62555778287744,4392 "n_items": 72,4393 "n_groups": 24394 },4395 "edit_sim": {4396 "mean": 0.6304469789152201,4397 "ci_low": 0.6170963361652347,4398 "ci_high": 0.6371223002902129,4399 "n_items": 72,4400 "n_groups": 24401 },4402 "rougeL": {4403 "mean": 0.6568634503601365,4404 "ci_low": 0.6567049789978942,4405 "ci_high": 0.6571803930846212,4406 "n_items": 72,4407 "n_groups": 24408 },4409 "exact_match": {4410 "mean": 0.0,4411 "ci_low": 0.0,4412 "ci_high": 0.0,4413 "n_items": 72,4414 "n_groups": 24415 },4416 "number_recall": {4417 "mean": 0.9930555555555556,4418 "ci_low": 0.9895833333333334,4419 "ci_high": 1.0,4420 "n_items": 72,4421 "n_groups": 24422 },4423 "placeholders_exact": {4424 "mean": 0.9861111111111112,4425 "ci_low": 0.9583333333333334,4426 "ci_high": 1.0,4427 "n_items": 72,4428 "n_groups": 24429 }4430 }4431 },4432 "by_era": {4433 "1980-1999": {4434 "n": 8,4435 "chrf": {4436 "mean": 71.19300352219418,4437 "ci_low": 70.48711117204634,4438 "ci_high": 72.36949077244056,4439 "n_items": 8,4440 "n_groups": 24441 },4442 "edit_sim": {4443 "mean": 0.6956053277796619,4444 "ci_low": 0.6792900889812683,4445 "ci_high": 0.7227973924436509,4446 "n_items": 8,4447 "n_groups": 24448 },4449 "rougeL": {4450 "mean": 0.6914341090807077,4451 "ci_low": 0.6736841504712101,4452 "ci_high": 0.7210173734298699,4453 "n_items": 8,4454 "n_groups": 24455 },4456 "exact_match": {4457 "mean": 0.0,4458 "ci_low": 0.0,4459 "ci_high": 0.0,4460 "n_items": 8,4461 "n_groups": 24462 },4463 "number_recall": {4464 "mean": 1.0,4465 "ci_low": 1.0,4466 "ci_high": 1.0,4467 "n_items": 8,4468 "n_groups": 24469 },4470 "placeholders_exact": {4471 "mean": 1.0,4472 "ci_low": 1.0,4473 "ci_high": 1.0,4474 "n_items": 8,4475 "n_groups": 24476 }4477 },4478 "2000-2009": {4479 "n": 89,4480 "chrf": {4481 "mean": 69.34309143097222,4482 "ci_low": 67.19423204768476,4483 "ci_high": 73.1309770619058,4484 "n_items": 89,4485 "n_groups": 74486 },4487 "edit_sim": {4488 "mean": 0.6582660539358618,4489 "ci_low": 0.6292004956236301,4490 "ci_high": 0.7075053990624772,4491 "n_items": 89,4492 "n_groups": 74493 },4494 "rougeL": {4495 "mean": 0.666974093668935,4496 "ci_low": 0.6389301168689387,4497 "ci_high": 0.716682420795659,4498 "n_items": 89,4499 "n_groups": 74500 },4501 "exact_match": {4502 "mean": 0.0,4503 "ci_low": 0.0,4504 "ci_high": 0.0,4505 "n_items": 89,4506 "n_groups": 74507 },4508 "number_recall": {4509 "mean": 0.9836142322097378,4510 "ci_low": 0.9502698047283177,4511 "ci_high": 1.0,4512 "n_items": 89,4513 "n_groups": 74514 },4515 "placeholders_exact": {4516 "mean": 0.9887640449438202,4517 "ci_low": 0.9732142857142857,4518 "ci_high": 1.0,4519 "n_items": 89,4520 "n_groups": 74521 }4522 },4523 "2010-2017": {4524 "n": 715,4525 "chrf": {4526 "mean": 70.77045746486827,4527 "ci_low": 69.86221866676075,4528 "ci_high": 71.76052814787248,4529 "n_items": 715,4530 "n_groups": 454531 },4532 "edit_sim": {4533 "mean": 0.6700037367303819,4534 "ci_low": 0.6577297100254802,4535 "ci_high": 0.6820883313665408,4536 "n_items": 715,4537 "n_groups": 454538 },4539 "rougeL": {4540 "mean": 0.6816108648576894,4541 "ci_low": 0.6685791655818031,4542 "ci_high": 0.6943261077350016,4543 "n_items": 715,4544 "n_groups": 454545 },4546 "exact_match": {4547 "mean": 0.0,4548 "ci_low": 0.0,4549 "ci_high": 0.0,4550 "n_items": 715,4551 "n_groups": 454552 },4553 "number_recall": {4554 "mean": 0.9894451356928352,4555 "ci_low": 0.9825357642013611,4556 "ci_high": 0.99514607184612,4557 "n_items": 715,4558 "n_groups": 454559 },4560 "placeholders_exact": {4561 "mean": 0.9958041958041958,4562 "ci_low": 0.984743813703693,4563 "ci_high": 1.0,4564 "n_items": 715,4565 "n_groups": 454566 }4567 }4568 },4569 "by_role": {4570 "abstract": {4571 "n": 83,4572 "chrf": {4573 "mean": 72.9290397830678,4574 "ci_low": 71.18231044010155,4575 "ci_high": 74.6438453730331,4576 "n_items": 83,4577 "n_groups": 154578 },4579 "edit_sim": {4580 "mean": 0.6916486541072198,4581 "ci_low": 0.6670265735424016,4582 "ci_high": 0.7161469914296529,4583 "n_items": 83,4584 "n_groups": 154585 },4586 "rougeL": {4587 "mean": 0.699423560113586,4588 "ci_low": 0.6713853342658451,4589 "ci_high": 0.7286819093936237,4590 "n_items": 83,4591 "n_groups": 154592 },4593 "exact_match": {4594 "mean": 0.0,4595 "ci_low": 0.0,4596 "ci_high": 0.0,4597 "n_items": 83,4598 "n_groups": 154599 },4600 "number_recall": {4601 "mean": 1.0,4602 "ci_low": 1.0,4603 "ci_high": 1.0,4604 "n_items": 83,4605 "n_groups": 154606 },4607 "placeholders_exact": {4608 "mean": 0.9879518072289156,4609 "ci_low": 0.9634146341463414,4610 "ci_high": 1.0,4611 "n_items": 83,4612 "n_groups": 154613 }4614 },4615 "background": {4616 "n": 58,4617 "chrf": {4618 "mean": 71.53771085433442,4619 "ci_low": 68.50547710281114,4620 "ci_high": 74.45530810928595,4621 "n_items": 58,4622 "n_groups": 74623 },4624 "edit_sim": {4625 "mean": 0.6878474726343861,4626 "ci_low": 0.6577802204734245,4627 "ci_high": 0.718790818173482,4628 "n_items": 58,4629 "n_groups": 74630 },4631 "rougeL": {4632 "mean": 0.6955053405727081,4633 "ci_low": 0.6695381540990625,4634 "ci_high": 0.7192556506840805,4635 "n_items": 58,4636 "n_groups": 74637 },4638 "exact_match": {4639 "mean": 0.0,4640 "ci_low": 0.0,4641 "ci_high": 0.0,4642 "n_items": 58,4643 "n_groups": 74644 },4645 "number_recall": {4646 "mean": 0.9566570881226053,4647 "ci_low": 0.9049286740558293,4648 "ci_high": 0.9990583804143126,4649 "n_items": 58,4650 "n_groups": 74651 },4652 "placeholders_exact": {4653 "mean": 1.0,4654 "ci_low": 1.0,4655 "ci_high": 1.0,4656 "n_items": 58,4657 "n_groups": 74658 }4659 },4660 "conclusion": {4661 "n": 47,4662 "chrf": {4663 "mean": 69.70793363220716,4664 "ci_low": 67.39973230880508,4665 "ci_high": 71.7628992599724,4666 "n_items": 47,4667 "n_groups": 84668 },4669 "edit_sim": {4670 "mean": 0.6815387727985998,4671 "ci_low": 0.6537535282829141,4672 "ci_high": 0.7089358759810703,4673 "n_items": 47,4674 "n_groups": 84675 },4676 "rougeL": {4677 "mean": 0.6781986843754496,4678 "ci_low": 0.6512494059365569,4679 "ci_high": 0.7009137853583561,4680 "n_items": 47,4681 "n_groups": 84682 },4683 "exact_match": {4684 "mean": 0.0,4685 "ci_low": 0.0,4686 "ci_high": 0.0,4687 "n_items": 47,4688 "n_groups": 84689 },4690 "number_recall": {4691 "mean": 0.9787234042553191,4692 "ci_low": 0.9361702127659575,4693 "ci_high": 1.0,4694 "n_items": 47,4695 "n_groups": 84696 },4697 "placeholders_exact": {4698 "mean": 1.0,4699 "ci_low": 1.0,4700 "ci_high": 1.0,4701 "n_items": 47,4702 "n_groups": 84703 }4704 },4705 "data": {4706 "n": 69,4707 "chrf": {4708 "mean": 71.97291357231994,4709 "ci_low": 69.3311558285489,4710 "ci_high": 75.03545133408475,4711 "n_items": 69,4712 "n_groups": 104713 },4714 "edit_sim": {4715 "mean": 0.64061911306256,4716 "ci_low": 0.5907311429894014,4717 "ci_high": 0.692294126693372,4718 "n_items": 69,4719 "n_groups": 104720 },4721 "rougeL": {4722 "mean": 0.6827935128580513,4723 "ci_low": 0.6432693157972297,4724 "ci_high": 0.7286296957564184,4725 "n_items": 69,4726 "n_groups": 104727 },4728 "exact_match": {4729 "mean": 0.0,4730 "ci_low": 0.0,4731 "ci_high": 0.0,4732 "n_items": 69,4733 "n_groups": 104734 },4735 "number_recall": {4736 "mean": 0.9846466780105224,4737 "ci_low": 0.9594226898582072,4738 "ci_high": 1.0,4739 "n_items": 69,4740 "n_groups": 104741 },4742 "placeholders_exact": {4743 "mean": 1.0,4744 "ci_low": 1.0,4745 "ci_high": 1.0,4746 "n_items": 69,4747 "n_groups": 104748 }4749 },4750 "discussion": {4751 "n": 103,4752 "chrf": {4753 "mean": 71.20064942337149,4754 "ci_low": 69.54729442858587,4755 "ci_high": 72.7927383583581,4756 "n_items": 103,4757 "n_groups": 144758 },4759 "edit_sim": {4760 "mean": 0.6940036888418125,4761 "ci_low": 0.6749812486934982,4762 "ci_high": 0.71476789375716,4763 "n_items": 103,4764 "n_groups": 144765 },4766 "rougeL": {4767 "mean": 0.6997586478156699,4768 "ci_low": 0.6727475668523399,4769 "ci_high": 0.7252192292048958,4770 "n_items": 103,4771 "n_groups": 144772 },4773 "exact_match": {4774 "mean": 0.0,4775 "ci_low": 0.0,4776 "ci_high": 0.0,4777 "n_items": 103,4778 "n_groups": 144779 },4780 "number_recall": {4781 "mean": 1.0,4782 "ci_low": 1.0,4783 "ci_high": 1.0,4784 "n_items": 103,4785 "n_groups": 144786 },4787 "placeholders_exact": {4788 "mean": 1.0,4789 "ci_low": 1.0,4790 "ci_high": 1.0,4791 "n_items": 103,4792 "n_groups": 144793 }4794 },4795 "introduction": {4796 "n": 99,4797 "chrf": {4798 "mean": 69.96284072188031,4799 "ci_low": 67.44330851425907,4800 "ci_high": 72.0846554980936,4801 "n_items": 99,4802 "n_groups": 144803 },4804 "edit_sim": {4805 "mean": 0.6579957215071831,4806 "ci_low": 0.6229172861286241,4807 "ci_high": 0.68911477036702,4808 "n_items": 99,4809 "n_groups": 144810 },4811 "rougeL": {4812 "mean": 0.6668917231457665,4813 "ci_low": 0.6411830330137043,4814 "ci_high": 0.6917192194994384,4815 "n_items": 99,4816 "n_groups": 144817 },4818 "exact_match": {4819 "mean": 0.0,4820 "ci_low": 0.0,4821 "ci_high": 0.0,4822 "n_items": 99,4823 "n_groups": 144824 },4825 "number_recall": {4826 "mean": 0.9959595959595959,4827 "ci_low": 0.9866666666666668,4828 "ci_high": 1.0,4829 "n_items": 99,4830 "n_groups": 144831 },4832 "placeholders_exact": {4833 "mean": 1.0,4834 "ci_low": 1.0,4835 "ci_high": 1.0,4836 "n_items": 99,4837 "n_groups": 144838 }4839 },4840 "limitations": {4841 "n": 42,4842 "chrf": {4843 "mean": 69.52271986808516,4844 "ci_low": 68.10695012663471,4845 "ci_high": 70.87773818129075,4846 "n_items": 42,4847 "n_groups": 44848 },4849 "edit_sim": {4850 "mean": 0.6592279379480658,4851 "ci_low": 0.6272188216236754,4852 "ci_high": 0.6812229790063463,4853 "n_items": 42,4854 "n_groups": 44855 },4856 "rougeL": {4857 "mean": 0.6624276013173471,4858 "ci_low": 0.6430149959295429,4859 "ci_high": 0.6769870553582003,4860 "n_items": 42,4861 "n_groups": 44862 },4863 "exact_match": {4864 "mean": 0.0,4865 "ci_low": 0.0,4866 "ci_high": 0.0,4867 "n_items": 42,4868 "n_groups": 44869 },4870 "number_recall": {4871 "mean": 0.9910714285714286,4872 "ci_low": 0.9765625,4873 "ci_high": 1.0,4874 "n_items": 42,4875 "n_groups": 44876 },4877 "placeholders_exact": {4878 "mean": 1.0,4879 "ci_low": 1.0,4880 "ci_high": 1.0,4881 "n_items": 42,4882 "n_groups": 44883 }4884 },4885 "methods": {4886 "n": 98,4887 "chrf": {4888 "mean": 70.40809937774121,4889 "ci_low": 67.08526468552684,4890 "ci_high": 74.02732723097839,4891 "n_items": 98,4892 "n_groups": 124893 },4894 "edit_sim": {4895 "mean": 0.6665640533414646,4896 "ci_low": 0.6420123622579605,4897 "ci_high": 0.6920566907177566,4898 "n_items": 98,4899 "n_groups": 124900 },4901 "rougeL": {4902 "mean": 0.6775968704360329,4903 "ci_low": 0.6495059060663554,4904 "ci_high": 0.7047940687834363,4905 "n_items": 98,4906 "n_groups": 124907 },4908 "exact_match": {4909 "mean": 0.0,4910 "ci_low": 0.0,4911 "ci_high": 0.0,4912 "n_items": 98,4913 "n_groups": 124914 },4915 "number_recall": {4916 "mean": 0.9967532467532468,4917 "ci_low": 0.9922394678492239,4918 "ci_high": 1.0,4919 "n_items": 98,4920 "n_groups": 124921 },4922 "placeholders_exact": {4923 "mean": 1.0,4924 "ci_low": 1.0,4925 "ci_high": 1.0,4926 "n_items": 98,4927 "n_groups": 124928 }4929 },4930 "other": {4931 "n": 70,4932 "chrf": {4933 "mean": 69.30945457357593,4934 "ci_low": 64.60319725564386,4935 "ci_high": 72.96321657370235,4936 "n_items": 70,4937 "n_groups": 94938 },4939 "edit_sim": {4940 "mean": 0.6651374217757287,4941 "ci_low": 0.6293073861134281,4942 "ci_high": 0.6941709951941707,4943 "n_items": 70,4944 "n_groups": 94945 },4946 "rougeL": {4947 "mean": 0.663313428385058,4948 "ci_low": 0.6104119207081795,4949 "ci_high": 0.7083721277039767,4950 "n_items": 70,4951 "n_groups": 94952 },4953 "exact_match": {4954 "mean": 0.0,4955 "ci_low": 0.0,4956 "ci_high": 0.0,4957 "n_items": 70,4958 "n_groups": 94959 },4960 "number_recall": {4961 "mean": 0.9971428571428571,4962 "ci_low": 0.9894736842105264,4963 "ci_high": 1.0,4964 "n_items": 70,4965 "n_groups": 94966 },4967 "placeholders_exact": {4968 "mean": 0.9571428571428572,4969 "ci_low": 0.8783783783783784,4970 "ci_high": 1.0,4971 "n_items": 70,4972 "n_groups": 94973 }4974 },4975 "related_work": {4976 "n": 6,4977 "chrf": {4978 "mean": 73.35482282783146,4979 "ci_low": 73.35482282783146,4980 "ci_high": 73.35482282783146,4981 "n_items": 6,4982 "n_groups": 14983 },4984 "edit_sim": {4985 "mean": 0.7085461524565435,4986 "ci_low": 0.7085461524565435,4987 "ci_high": 0.7085461524565435,4988 "n_items": 6,4989 "n_groups": 14990 },4991 "rougeL": {4992 "mean": 0.7208290740567366,4993 "ci_low": 0.7208290740567366,4994 "ci_high": 0.7208290740567366,4995 "n_items": 6,4996 "n_groups": 14997 },4998 "exact_match": {4999 "mean": 0.0,5000 "ci_low": 0.0,5001 "ci_high": 0.0,5002 "n_items": 6,5003 "n_groups": 15004 },5005 "number_recall": {5006 "mean": 1.0,5007 "ci_low": 1.0,5008 "ci_high": 1.0,5009 "n_items": 6,5010 "n_groups": 15011 },5012 "placeholders_exact": {5013 "mean": 1.0,5014 "ci_low": 1.0,5015 "ci_high": 1.0,5016 "n_items": 6,5017 "n_groups": 15018 }5019 },5020 "results": {5021 "n": 78,5022 "chrf": {5023 "mean": 68.48815667849847,5024 "ci_low": 67.61642981083584,5025 "ci_high": 69.36058238284022,5026 "n_items": 78,5027 "n_groups": 95028 },5029 "edit_sim": {5030 "mean": 0.63268445435528,5031 "ci_low": 0.5977265396106627,5032 "ci_high": 0.6651438540625536,5033 "n_items": 78,5034 "n_groups": 95035 },5036 "rougeL": {5037 "mean": 0.6584782227093783,5038 "ci_low": 0.6364705051077433,5039 "ci_high": 0.6787643739410586,5040 "n_items": 78,5041 "n_groups": 95042 },5043 "exact_match": {5044 "mean": 0.0,5045 "ci_low": 0.0,5046 "ci_high": 0.0,5047 "n_items": 78,5048 "n_groups": 95049 },5050 "number_recall": {5051 "mean": 0.9922161172161171,5052 "ci_low": 0.9848478084415584,5053 "ci_high": 1.0,5054 "n_items": 78,5055 "n_groups": 95056 },5057 "placeholders_exact": {5058 "mean": 1.0,5059 "ci_low": 1.0,5060 "ci_high": 1.0,5061 "n_items": 78,5062 "n_groups": 95063 }5064 },5065 "theory": {5066 "n": 59,5067 "chrf": {5068 "mean": 70.90596153513972,5069 "ci_low": 69.67501591683339,5070 "ci_high": 72.94741453188207,5071 "n_items": 59,5072 "n_groups": 65073 },5074 "edit_sim": {5075 "mean": 0.6757808386144987,5076 "ci_low": 0.6575896121783434,5077 "ci_high": 0.6970909955522084,5078 "n_items": 59,5079 "n_groups": 65080 },5081 "rougeL": {5082 "mean": 0.6851235611830798,5083 "ci_low": 0.6697272883332718,5084 "ci_high": 0.7021048484485835,5085 "n_items": 59,5086 "n_groups": 65087 },5088 "exact_match": {5089 "mean": 0.0,5090 "ci_low": 0.0,5091 "ci_high": 0.0,5092 "n_items": 59,5093 "n_groups": 65094 },5095 "number_recall": {5096 "mean": 0.9570937537039231,5097 "ci_low": 0.8892971435344317,5098 "ci_high": 1.0,5099 "n_items": 59,5100 "n_groups": 65101 },5102 "placeholders_exact": {5103 "mean": 1.0,5104 "ci_low": 1.0,5105 "ci_high": 1.0,5106 "n_items": 59,5107 "n_groups": 65108 }5109 }5110 }5111 },5112 "htr-m1-tiny": {5113 "overall": {5114 "n": 812,5115 "chrf": {5116 "mean": 70.6614480618171,5117 "ci_low": 69.8431331916207,5118 "ci_high": 71.62075541163328,5119 "n_items": 812,5120 "n_groups": 545121 },5122 "edit_sim": {5123 "mean": 0.669130864010782,5124 "ci_low": 0.6578901218291514,5125 "ci_high": 0.6803616556716624,5126 "n_items": 812,5127 "n_groups": 545128 },5129 "rougeL": {5130 "mean": 0.6807568170852932,5131 "ci_low": 0.6701751573330922,5132 "ci_high": 0.6923159985894061,5133 "n_items": 812,5134 "n_groups": 545135 },5136 "exact_match": {5137 "mean": 0.0,5138 "ci_low": 0.0,5139 "ci_high": 0.0,5140 "n_items": 812,5141 "n_groups": 545142 },5143 "number_recall": {5144 "mean": 0.9912125607014769,5145 "ci_low": 0.9861340158635069,5146 "ci_high": 0.9952634539784347,5147 "n_items": 812,5148 "n_groups": 545149 },5150 "placeholders_exact": {5151 "mean": 0.9889162561576355,5152 "ci_low": 0.975418765454712,5153 "ci_high": 0.9972827008731049,5154 "n_items": 812,5155 "n_groups": 545156 }5157 },5158 "by_variant": {5159 "A": {5160 "n": 133,5161 "chrf": {5162 "mean": 69.5336141947682,5163 "ci_low": 68.6451232591013,5164 "ci_high": 70.41550181104314,5165 "n_items": 133,5166 "n_groups": 505167 },5168 "edit_sim": {5169 "mean": 0.6386633783296437,5170 "ci_low": 0.6246878438291226,5171 "ci_high": 0.6505900392970699,5172 "n_items": 133,5173 "n_groups": 505174 },5175 "rougeL": {5176 "mean": 0.6528562354060006,5177 "ci_low": 0.6404662157167831,5178 "ci_high": 0.6646216692922626,5179 "n_items": 133,5180 "n_groups": 505181 },5182 "exact_match": {5183 "mean": 0.0,5184 "ci_low": 0.0,5185 "ci_high": 0.0,5186 "n_items": 133,5187 "n_groups": 505188 },5189 "number_recall": {5190 "mean": 0.996732903687791,5191 "ci_low": 0.9924083458111507,5192 "ci_high": 1.0,5193 "n_items": 133,5194 "n_groups": 505195 },5196 "placeholders_exact": {5197 "mean": 0.9849624060150376,5198 "ci_low": 0.9596774193548387,5199 "ci_high": 1.0,5200 "n_items": 133,5201 "n_groups": 505202 }5203 },5204 "B": {5205 "n": 136,5206 "chrf": {5207 "mean": 75.87624743316617,5208 "ci_low": 74.74345959897764,5209 "ci_high": 77.11317966800192,5210 "n_items": 136,5211 "n_groups": 525212 },5213 "edit_sim": {5214 "mean": 0.7462575498465793,5215 "ci_low": 0.731379250888179,5216 "ci_high": 0.759366193666555,5217 "n_items": 136,5218 "n_groups": 525219 },5220 "rougeL": {5221 "mean": 0.7508178091565916,5222 "ci_low": 0.7372569283629397,5223 "ci_high": 0.7653016249779035,5224 "n_items": 136,5225 "n_groups": 525226 },5227 "exact_match": {5228 "mean": 0.0,5229 "ci_low": 0.0,5230 "ci_high": 0.0,5231 "n_items": 136,5232 "n_groups": 525233 },5234 "number_recall": {5235 "mean": 0.9923666815210933,5236 "ci_low": 0.9796251578282829,5237 "ci_high": 0.9990788897927666,5238 "n_items": 136,5239 "n_groups": 525240 },5241 "placeholders_exact": {5242 "mean": 0.9926470588235294,5243 "ci_low": 0.9734513274336283,5244 "ci_high": 1.0,5245 "n_items": 136,5246 "n_groups": 525247 }5248 },5249 "C": {5250 "n": 135,5251 "chrf": {5252 "mean": 72.2154451860098,5253 "ci_low": 71.14883438368517,5254 "ci_high": 73.49793444859846,5255 "n_items": 135,5256 "n_groups": 535257 },5258 "edit_sim": {5259 "mean": 0.7175533062015219,5260 "ci_low": 0.7047612975321351,5261 "ci_high": 0.7311181690380277,5262 "n_items": 135,5263 "n_groups": 535264 },5265 "rougeL": {5266 "mean": 0.7347263570548849,5267 "ci_low": 0.7219964028467647,5268 "ci_high": 0.7502041776781442,5269 "n_items": 135,5270 "n_groups": 535271 },5272 "exact_match": {5273 "mean": 0.0,5274 "ci_low": 0.0,5275 "ci_high": 0.0,5276 "n_items": 135,5277 "n_groups": 535278 },5279 "number_recall": {5280 "mean": 0.9870901320901321,5281 "ci_low": 0.9719739640465742,5282 "ci_high": 0.9967175430778371,5283 "n_items": 135,5284 "n_groups": 535285 },5286 "placeholders_exact": {5287 "mean": 0.9925925925925926,5288 "ci_low": 0.974570838765754,5289 "ci_high": 1.0,5290 "n_items": 135,5291 "n_groups": 535292 }5293 },5294 "D": {5295 "n": 136,5296 "chrf": {5297 "mean": 68.0839185046232,5298 "ci_low": 67.05638325728036,5299 "ci_high": 69.110308935979,5300 "n_items": 136,5301 "n_groups": 515302 },5303 "edit_sim": {5304 "mean": 0.6197068749353221,5305 "ci_low": 0.6047652347344094,5306 "ci_high": 0.6348204425238742,5307 "n_items": 136,5308 "n_groups": 515309 },5310 "rougeL": {5311 "mean": 0.6329865837841483,5312 "ci_low": 0.6193201873902621,5313 "ci_high": 0.6468817285905863,5314 "n_items": 136,5315 "n_groups": 515316 },5317 "exact_match": {5318 "mean": 0.0,5319 "ci_low": 0.0,5320 "ci_high": 0.0,5321 "n_items": 136,5322 "n_groups": 515323 },5324 "number_recall": {5325 "mean": 0.9853522090654444,5326 "ci_low": 0.9691682208192823,5327 "ci_high": 0.9969019079376222,5328 "n_items": 136,5329 "n_groups": 515330 },5331 "placeholders_exact": {5332 "mean": 0.9705882352941176,5333 "ci_low": 0.9385964912280702,5334 "ci_high": 0.9931984280198566,5335 "n_items": 136,5336 "n_groups": 515337 }5338 },5339 "E": {5340 "n": 272,5341 "chrf": {5342 "mean": 69.12300540676705,5343 "ci_low": 68.20301283434948,5344 "ci_high": 70.18245746275859,5345 "n_items": 272,5346 "n_groups": 525347 },5348 "edit_sim": {5349 "mean": 0.6461440226153248,5350 "ci_low": 0.6321644968304002,5351 "ci_high": 0.6623819304610052,5352 "n_items": 272,5353 "n_groups": 525354 },5355 "rougeL": {5356 "mean": 0.65646764897761,5357 "ci_low": 0.6428652644750523,5358 "ci_high": 0.6712941103695574,5359 "n_items": 272,5360 "n_groups": 525361 },5362 "exact_match": {5363 "mean": 0.0,5364 "ci_low": 0.0,5365 "ci_high": 0.0,5366 "n_items": 272,5367 "n_groups": 525368 },5369 "number_recall": {5370 "mean": 0.9929124490705372,5371 "ci_low": 0.9860406068043689,5372 "ci_high": 0.9979595017312662,5373 "n_items": 272,5374 "n_groups": 525375 },5376 "placeholders_exact": {5377 "mean": 0.9963235294117647,5378 "ci_low": 0.9871227273358611,5379 "ci_high": 1.0,5380 "n_items": 272,5381 "n_groups": 525382 }5383 }5384 },5385 "by_discipline": {5386 "biomedicine": {5387 "n": 380,5388 "chrf": {5389 "mean": 70.92674661694747,5390 "ci_low": 69.59141900174052,5391 "ci_high": 72.1355192911183,5392 "n_items": 380,5393 "n_groups": 315394 },5395 "edit_sim": {5396 "mean": 0.6640462304786949,5397 "ci_low": 0.6473493407305512,5398 "ci_high": 0.6757945681057222,5399 "n_items": 380,5400 "n_groups": 315401 },5402 "rougeL": {5403 "mean": 0.6801078223316999,5404 "ci_low": 0.6646044022938611,5405 "ci_high": 0.6917041406567597,5406 "n_items": 380,5407 "n_groups": 315408 },5409 "exact_match": {5410 "mean": 0.0,5411 "ci_low": 0.0,5412 "ci_high": 0.0,5413 "n_items": 380,5414 "n_groups": 315415 },5416 "number_recall": {5417 "mean": 0.9954615705931494,5418 "ci_low": 0.989605249441698,5419 "ci_high": 0.9989213859057251,5420 "n_items": 380,5421 "n_groups": 315422 },5423 "placeholders_exact": {5424 "mean": 0.9947368421052631,5425 "ci_low": 0.9885044198455045,5426 "ci_high": 1.0,5427 "n_items": 380,5428 "n_groups": 315429 }5430 },5431 "engineering": {5432 "n": 12,5433 "chrf": {5434 "mean": 73.19187434159339,5435 "ci_low": 72.45502418434774,5436 "ci_high": 73.92872449883906,5437 "n_items": 12,5438 "n_groups": 25439 },5440 "edit_sim": {5441 "mean": 0.7100347558557764,5442 "ci_low": 0.702713251636324,5443 "ci_high": 0.7173562600752285,5444 "n_items": 12,5445 "n_groups": 25446 },5447 "rougeL": {5448 "mean": 0.6964777179955997,5449 "ci_low": 0.6662575140583972,5450 "ci_high": 0.7266979219328022,5451 "n_items": 12,5452 "n_groups": 25453 },5454 "exact_match": {5455 "mean": 0.0,5456 "ci_low": 0.0,5457 "ci_high": 0.0,5458 "n_items": 12,5459 "n_groups": 25460 },5461 "number_recall": {5462 "mean": 1.0,5463 "ci_low": 1.0,5464 "ci_high": 1.0,5465 "n_items": 12,5466 "n_groups": 25467 },5468 "placeholders_exact": {5469 "mean": 0.9166666666666666,5470 "ci_low": 0.8333333333333334,5471 "ci_high": 1.0,5472 "n_items": 12,5473 "n_groups": 25474 }5475 },5476 "finance": {5477 "n": 96,5478 "chrf": {5479 "mean": 69.26341825311175,5480 "ci_low": 69.26341825311175,5481 "ci_high": 69.26341825311175,5482 "n_items": 96,5483 "n_groups": 15484 },5485 "edit_sim": {5486 "mean": 0.6604153615950016,5487 "ci_low": 0.6604153615950016,5488 "ci_high": 0.6604153615950016,5489 "n_items": 96,5490 "n_groups": 15491 },5492 "rougeL": {5493 "mean": 0.6583863163505422,5494 "ci_low": 0.6583863163505422,5495 "ci_high": 0.6583863163505422,5496 "n_items": 96,5497 "n_groups": 15498 },5499 "exact_match": {5500 "mean": 0.0,5501 "ci_low": 0.0,5502 "ci_high": 0.0,5503 "n_items": 96,5504 "n_groups": 15505 },5506 "number_recall": {5507 "mean": 0.998263888888889,5508 "ci_low": 0.998263888888889,5509 "ci_high": 0.998263888888889,5510 "n_items": 96,5511 "n_groups": 15512 },5513 "placeholders_exact": {5514 "mean": 1.0,5515 "ci_low": 1.0,5516 "ci_high": 1.0,5517 "n_items": 96,5518 "n_groups": 15519 }5520 },5521 "life_sciences": {5522 "n": 162,5523 "chrf": {5524 "mean": 71.83546047577236,5525 "ci_low": 69.77653371826143,5526 "ci_high": 73.98167928127488,5527 "n_items": 162,5528 "n_groups": 155529 },5530 "edit_sim": {5531 "mean": 0.6802716571348165,5532 "ci_low": 0.661044709524179,5533 "ci_high": 0.7037330163147154,5534 "n_items": 162,5535 "n_groups": 155536 },5537 "rougeL": {5538 "mean": 0.6989932236400936,5539 "ci_low": 0.6755156105686282,5540 "ci_high": 0.7263882275789134,5541 "n_items": 162,5542 "n_groups": 155543 },5544 "exact_match": {5545 "mean": 0.0,5546 "ci_low": 0.0,5547 "ci_high": 0.0,5548 "n_items": 162,5549 "n_groups": 155550 },5551 "number_recall": {5552 "mean": 0.9820321139765584,5553 "ci_low": 0.9697889322463585,5554 "ci_high": 0.9953178404928404,5555 "n_items": 162,5556 "n_groups": 155557 },5558 "placeholders_exact": {5559 "mean": 0.9753086419753086,5560 "ci_low": 0.9172413793103448,5561 "ci_high": 1.0,5562 "n_items": 162,5563 "n_groups": 155564 }5565 },5566 "mathematics": {5567 "n": 60,5568 "chrf": {5569 "mean": 70.18668701701932,5570 "ci_low": 70.18668701701932,5571 "ci_high": 70.18668701701932,5572 "n_items": 60,5573 "n_groups": 15574 },5575 "edit_sim": {5576 "mean": 0.7065190662092039,5577 "ci_low": 0.7065190662092039,5578 "ci_high": 0.7065190662092039,5579 "n_items": 60,5580 "n_groups": 15581 },5582 "rougeL": {5583 "mean": 0.6935769538757255,5584 "ci_low": 0.6935769538757255,5585 "ci_high": 0.6935769538757255,5586 "n_items": 60,5587 "n_groups": 15588 },5589 "exact_match": {5590 "mean": 0.0,5591 "ci_low": 0.0,5592 "ci_high": 0.0,5593 "n_items": 60,5594 "n_groups": 15595 },5596 "number_recall": {5597 "mean": 0.9833333333333333,5598 "ci_low": 0.9833333333333333,5599 "ci_high": 0.9833333333333333,5600 "n_items": 60,5601 "n_groups": 15602 },5603 "placeholders_exact": {5604 "mean": 0.9833333333333333,5605 "ci_low": 0.9833333333333333,5606 "ci_high": 0.9833333333333333,5607 "n_items": 60,5608 "n_groups": 15609 }5610 },5611 "social_sciences": {5612 "n": 30,5613 "chrf": {5614 "mean": 69.02741362174889,5615 "ci_low": 56.47122868313755,5616 "ci_high": 72.16645985640173,5617 "n_items": 30,5618 "n_groups": 25619 },5620 "edit_sim": {5621 "mean": 0.6951346952982493,5622 "ci_low": 0.5892244026940571,5623 "ci_high": 0.7216122684492975,5624 "n_items": 30,5625 "n_groups": 25626 },5627 "rougeL": {5628 "mean": 0.6798743533608529,5629 "ci_low": 0.48862219300600773,5630 "ci_high": 0.727687393449564,5631 "n_items": 30,5632 "n_groups": 25633 },5634 "exact_match": {5635 "mean": 0.0,5636 "ci_low": 0.0,5637 "ci_high": 0.0,5638 "n_items": 30,5639 "n_groups": 25640 },5641 "number_recall": {5642 "mean": 0.9777777777777777,5643 "ci_low": 0.9722222222222223,5644 "ci_high": 1.0,5645 "n_items": 30,5646 "n_groups": 25647 },5648 "placeholders_exact": {5649 "mean": 1.0,5650 "ci_low": 1.0,5651 "ci_high": 1.0,5652 "n_items": 30,5653 "n_groups": 25654 }5655 },5656 "statistics": {5657 "n": 72,5658 "chrf": {5659 "mean": 69.13851723067847,5660 "ci_low": 67.19136632720766,5661 "ci_high": 70.1120926824139,5662 "n_items": 72,5663 "n_groups": 25664 },5665 "edit_sim": {5666 "mean": 0.6337112350572427,5667 "ci_low": 0.6130899846755188,5668 "ci_high": 0.6440218602481049,5669 "n_items": 72,5670 "n_groups": 25671 },5672 "rougeL": {5673 "mean": 0.6600415824798413,5674 "ci_low": 0.6563486867986156,5675 "ci_high": 0.6618880303204543,5676 "n_items": 72,5677 "n_groups": 25678 },5679 "exact_match": {5680 "mean": 0.0,5681 "ci_low": 0.0,5682 "ci_high": 0.0,5683 "n_items": 72,5684 "n_groups": 25685 },5686 "number_recall": {5687 "mean": 0.9907407407407407,5688 "ci_low": 0.986111111111111,5689 "ci_high": 1.0,5690 "n_items": 72,5691 "n_groups": 25692 },5693 "placeholders_exact": {5694 "mean": 0.9861111111111112,5695 "ci_low": 0.9583333333333334,5696 "ci_high": 1.0,5697 "n_items": 72,5698 "n_groups": 25699 }5700 }5701 },5702 "by_era": {5703 "1980-1999": {5704 "n": 8,5705 "chrf": {5706 "mean": 71.30929249015135,5707 "ci_low": 70.50412995786571,5708 "ci_high": 72.65123004396075,5709 "n_items": 8,5710 "n_groups": 25711 },5712 "edit_sim": {5713 "mean": 0.7009860597228509,5714 "ci_low": 0.6876347381838352,5715 "ci_high": 0.723238262287877,5716 "n_items": 8,5717 "n_groups": 25718 },5719 "rougeL": {5720 "mean": 0.6981618078545136,5721 "ci_low": 0.6781285554779249,5722 "ci_high": 0.7315505618154949,5723 "n_items": 8,5724 "n_groups": 25725 },5726 "exact_match": {5727 "mean": 0.0,5728 "ci_low": 0.0,5729 "ci_high": 0.0,5730 "n_items": 8,5731 "n_groups": 25732 },5733 "number_recall": {5734 "mean": 1.0,5735 "ci_low": 1.0,5736 "ci_high": 1.0,5737 "n_items": 8,5738 "n_groups": 25739 },5740 "placeholders_exact": {5741 "mean": 1.0,5742 "ci_low": 1.0,5743 "ci_high": 1.0,5744 "n_items": 8,5745 "n_groups": 25746 }5747 },5748 "2000-2009": {5749 "n": 89,5750 "chrf": {5751 "mean": 69.1102688538921,5752 "ci_low": 67.43529317349747,5753 "ci_high": 72.12950053733088,5754 "n_items": 89,5755 "n_groups": 75756 },5757 "edit_sim": {5758 "mean": 0.6523611335381058,5759 "ci_low": 0.6274826339868284,5760 "ci_high": 0.6967475406257716,5761 "n_items": 89,5762 "n_groups": 75763 },5764 "rougeL": {5765 "mean": 0.6636642220018959,5766 "ci_low": 0.6403520078001067,5767 "ci_high": 0.707431613139358,5768 "n_items": 89,5769 "n_groups": 75770 },5771 "exact_match": {5772 "mean": 0.0,5773 "ci_low": 0.0,5774 "ci_high": 0.0,5775 "n_items": 89,5776 "n_groups": 75777 },5778 "number_recall": {5779 "mean": 0.9859550561797753,5780 "ci_low": 0.9609375,5781 "ci_high": 1.0,5782 "n_items": 89,5783 "n_groups": 75784 },5785 "placeholders_exact": {5786 "mean": 0.9887640449438202,5787 "ci_low": 0.9732142857142857,5788 "ci_high": 1.0,5789 "n_items": 89,5790 "n_groups": 75791 }5792 },5793 "2010-2017": {5794 "n": 715,5795 "chrf": {5796 "mean": 70.84728329829073,5797 "ci_low": 69.92319844760168,5798 "ci_high": 71.8857548920812,5799 "n_items": 715,5800 "n_groups": 455801 },5802 "edit_sim": {5803 "mean": 0.6708618632364766,5804 "ci_low": 0.659357853800647,5805 "ci_high": 0.68271017156164,5806 "n_items": 715,5807 "n_groups": 455808 },5809 "rougeL": {5810 "mean": 0.682689685667487,5811 "ci_low": 0.6707565649375237,5812 "ci_high": 0.6946071552091955,5813 "n_items": 715,5814 "n_groups": 455815 },5816 "exact_match": {5817 "mean": 0.0,5818 "ci_low": 0.0,5819 "ci_high": 0.0,5820 "n_items": 715,5821 "n_groups": 455822 },5823 "number_recall": {5824 "mean": 0.991768670335104,5825 "ci_low": 0.9866311169963824,5826 "ci_high": 0.9957485480065118,5827 "n_items": 715,5828 "n_groups": 455829 },5830 "placeholders_exact": {5831 "mean": 0.9888111888111888,5832 "ci_low": 0.9738892347861016,5833 "ci_high": 0.9982734000601249,5834 "n_items": 715,5835 "n_groups": 455836 }5837 }5838 },5839 "by_role": {5840 "abstract": {5841 "n": 83,5842 "chrf": {5843 "mean": 73.26755440784918,5844 "ci_low": 71.76429985303527,5845 "ci_high": 74.68917021563631,5846 "n_items": 83,5847 "n_groups": 155848 },5849 "edit_sim": {5850 "mean": 0.6961556019516171,5851 "ci_low": 0.6710835738701532,5852 "ci_high": 0.7191869779179894,5853 "n_items": 83,5854 "n_groups": 155855 },5856 "rougeL": {5857 "mean": 0.7044507516607835,5858 "ci_low": 0.6774652773424605,5859 "ci_high": 0.7319885229504444,5860 "n_items": 83,5861 "n_groups": 155862 },5863 "exact_match": {5864 "mean": 0.0,5865 "ci_low": 0.0,5866 "ci_high": 0.0,5867 "n_items": 83,5868 "n_groups": 155869 },5870 "number_recall": {5871 "mean": 1.0,5872 "ci_low": 1.0,5873 "ci_high": 1.0,5874 "n_items": 83,5875 "n_groups": 155876 },5877 "placeholders_exact": {5878 "mean": 0.9879518072289156,5879 "ci_low": 0.9634146341463414,5880 "ci_high": 1.0,5881 "n_items": 83,5882 "n_groups": 155883 }5884 },5885 "background": {5886 "n": 58,5887 "chrf": {5888 "mean": 71.73431361557171,5889 "ci_low": 68.77421675082081,5890 "ci_high": 74.5778298499997,5891 "n_items": 58,5892 "n_groups": 75893 },5894 "edit_sim": {5895 "mean": 0.6881122963940343,5896 "ci_low": 0.6591155064966104,5897 "ci_high": 0.7179541654919324,5898 "n_items": 58,5899 "n_groups": 75900 },5901 "rougeL": {5902 "mean": 0.7000515889642552,5903 "ci_low": 0.6723594038984615,5904 "ci_high": 0.7236090847351184,5905 "n_items": 58,5906 "n_groups": 75907 },5908 "exact_match": {5909 "mean": 0.0,5910 "ci_low": 0.0,5911 "ci_high": 0.0,5912 "n_items": 58,5913 "n_groups": 75914 },5915 "number_recall": {5916 "mean": 0.9818007662835249,5917 "ci_low": 0.9507168458781362,5918 "ci_high": 1.0,5919 "n_items": 58,5920 "n_groups": 75921 },5922 "placeholders_exact": {5923 "mean": 1.0,5924 "ci_low": 1.0,5925 "ci_high": 1.0,5926 "n_items": 58,5927 "n_groups": 75928 }5929 },5930 "conclusion": {5931 "n": 47,5932 "chrf": {5933 "mean": 69.26619568541948,5934 "ci_low": 66.74266034203265,5935 "ci_high": 71.25464771674685,5936 "n_items": 47,5937 "n_groups": 85938 },5939 "edit_sim": {5940 "mean": 0.6732199091544626,5941 "ci_low": 0.6419380528157239,5942 "ci_high": 0.7044990206827454,5943 "n_items": 47,5944 "n_groups": 85945 },5946 "rougeL": {5947 "mean": 0.6731349186380692,5948 "ci_low": 0.647659132830888,5949 "ci_high": 0.6945698484189463,5950 "n_items": 47,5951 "n_groups": 85952 },5953 "exact_match": {5954 "mean": 0.0,5955 "ci_low": 0.0,5956 "ci_high": 0.0,5957 "n_items": 47,5958 "n_groups": 85959 },5960 "number_recall": {5961 "mean": 1.0,5962 "ci_low": 1.0,5963 "ci_high": 1.0,5964 "n_items": 47,5965 "n_groups": 85966 },5967 "placeholders_exact": {5968 "mean": 1.0,5969 "ci_low": 1.0,5970 "ci_high": 1.0,5971 "n_items": 47,5972 "n_groups": 85973 }5974 },5975 "data": {5976 "n": 69,5977 "chrf": {5978 "mean": 71.92835011659811,5979 "ci_low": 69.26986445918928,5980 "ci_high": 74.89995972487344,5981 "n_items": 69,5982 "n_groups": 105983 },5984 "edit_sim": {5985 "mean": 0.6419219646663883,5986 "ci_low": 0.5914780179440551,5987 "ci_high": 0.6911323931140215,5988 "n_items": 69,5989 "n_groups": 105990 },5991 "rougeL": {5992 "mean": 0.684827542103796,5993 "ci_low": 0.6449205742207752,5994 "ci_high": 0.7264192246626643,5995 "n_items": 69,5996 "n_groups": 105997 },5998 "exact_match": {5999 "mean": 0.0,6000 "ci_low": 0.0,6001 "ci_high": 0.0,6002 "n_items": 69,6003 "n_groups": 106004 },6005 "number_recall": {6006 "mean": 0.9861111111111109,6007 "ci_low": 0.9620646766169155,6008 "ci_high": 1.0,6009 "n_items": 69,6010 "n_groups": 106011 },6012 "placeholders_exact": {6013 "mean": 1.0,6014 "ci_low": 1.0,6015 "ci_high": 1.0,6016 "n_items": 69,6017 "n_groups": 106018 }6019 },6020 "discussion": {6021 "n": 103,6022 "chrf": {6023 "mean": 71.19618481512094,6024 "ci_low": 69.47329993143596,6025 "ci_high": 73.09583546269923,6026 "n_items": 103,6027 "n_groups": 146028 },6029 "edit_sim": {6030 "mean": 0.6912768422705522,6031 "ci_low": 0.6701470628497919,6032 "ci_high": 0.7164275896212131,6033 "n_items": 103,6034 "n_groups": 146035 },6036 "rougeL": {6037 "mean": 0.6979963983655316,6038 "ci_low": 0.6722095602273236,6039 "ci_high": 0.727339712801851,6040 "n_items": 103,6041 "n_groups": 146042 },6043 "exact_match": {6044 "mean": 0.0,6045 "ci_low": 0.0,6046 "ci_high": 0.0,6047 "n_items": 103,6048 "n_groups": 146049 },6050 "number_recall": {6051 "mean": 0.9856680536292186,6052 "ci_low": 0.9675019217542582,6053 "ci_high": 1.0,6054 "n_items": 103,6055 "n_groups": 146056 },6057 "placeholders_exact": {6058 "mean": 0.9902912621359223,6059 "ci_low": 0.9666666666666667,6060 "ci_high": 1.0,6061 "n_items": 103,6062 "n_groups": 146063 }6064 },6065 "introduction": {6066 "n": 99,6067 "chrf": {6068 "mean": 70.04010269555499,6069 "ci_low": 67.75157567051451,6070 "ci_high": 71.95587776541771,6071 "n_items": 99,6072 "n_groups": 146073 },6074 "edit_sim": {6075 "mean": 0.6606161247292697,6076 "ci_low": 0.6287633000359077,6077 "ci_high": 0.6889601130651338,6078 "n_items": 99,6079 "n_groups": 146080 },6081 "rougeL": {6082 "mean": 0.6679642063981734,6083 "ci_low": 0.645835815239391,6084 "ci_high": 0.689051190218394,6085 "n_items": 99,6086 "n_groups": 146087 },6088 "exact_match": {6089 "mean": 0.0,6090 "ci_low": 0.0,6091 "ci_high": 0.0,6092 "n_items": 99,6093 "n_groups": 146094 },6095 "number_recall": {6096 "mean": 0.993939393939394,6097 "ci_low": 0.9800000000000002,6098 "ci_high": 1.0,6099 "n_items": 99,6100 "n_groups": 146101 },6102 "placeholders_exact": {6103 "mean": 0.98989898989899,6104 "ci_low": 0.9655172413793104,6105 "ci_high": 1.0,6106 "n_items": 99,6107 "n_groups": 146108 }6109 },6110 "limitations": {6111 "n": 42,6112 "chrf": {6113 "mean": 69.56199587174713,6114 "ci_low": 67.96804828836825,6115 "ci_high": 71.44205395811292,6116 "n_items": 42,6117 "n_groups": 46118 },6119 "edit_sim": {6120 "mean": 0.6624896872473927,6121 "ci_low": 0.6309480811343106,6122 "ci_high": 0.6849736843807896,6123 "n_items": 42,6124 "n_groups": 46125 },6126 "rougeL": {6127 "mean": 0.6606235549084032,6128 "ci_low": 0.6500571515396714,6129 "ci_high": 0.6712057794588743,6130 "n_items": 42,6131 "n_groups": 46132 },6133 "exact_match": {6134 "mean": 0.0,6135 "ci_low": 0.0,6136 "ci_high": 0.0,6137 "n_items": 42,6138 "n_groups": 46139 },6140 "number_recall": {6141 "mean": 1.0,6142 "ci_low": 1.0,6143 "ci_high": 1.0,6144 "n_items": 42,6145 "n_groups": 46146 },6147 "placeholders_exact": {6148 "mean": 1.0,6149 "ci_low": 1.0,6150 "ci_high": 1.0,6151 "n_items": 42,6152 "n_groups": 46153 }6154 },6155 "methods": {6156 "n": 98,6157 "chrf": {6158 "mean": 70.44000590192762,6159 "ci_low": 67.23140015908362,6160 "ci_high": 74.03908622362677,6161 "n_items": 98,6162 "n_groups": 126163 },6164 "edit_sim": {6165 "mean": 0.6640282196283224,6166 "ci_low": 0.6404105499059835,6167 "ci_high": 0.687698346653146,6168 "n_items": 98,6169 "n_groups": 126170 },6171 "rougeL": {6172 "mean": 0.6795721087327061,6173 "ci_low": 0.6536275986264658,6174 "ci_high": 0.7072042569774573,6175 "n_items": 98,6176 "n_groups": 126177 },6178 "exact_match": {6179 "mean": 0.0,6180 "ci_low": 0.0,6181 "ci_high": 0.0,6182 "n_items": 98,6183 "n_groups": 126184 },6185 "number_recall": {6186 "mean": 0.9948979591836735,6187 "ci_low": 0.9825581395348836,6188 "ci_high": 1.0,6189 "n_items": 98,6190 "n_groups": 126191 },6192 "placeholders_exact": {6193 "mean": 0.9897959183673469,6194 "ci_low": 0.961038961038961,6195 "ci_high": 1.0,6196 "n_items": 98,6197 "n_groups": 126198 }6199 },6200 "other": {6201 "n": 70,6202 "chrf": {6203 "mean": 68.81984258481323,6204 "ci_low": 64.13191040968894,6205 "ci_high": 72.69423671790366,6206 "n_items": 70,6207 "n_groups": 96208 },6209 "edit_sim": {6210 "mean": 0.6605287947054995,6211 "ci_low": 0.6242265377504037,6212 "ci_high": 0.6937082720754802,6213 "n_items": 70,6214 "n_groups": 96215 },6216 "rougeL": {6217 "mean": 0.6578150116310036,6218 "ci_low": 0.6025738372303224,6219 "ci_high": 0.706814802497678,6220 "n_items": 70,6221 "n_groups": 96222 },6223 "exact_match": {6224 "mean": 0.0,6225 "ci_low": 0.0,6226 "ci_high": 0.0,6227 "n_items": 70,6228 "n_groups": 96229 },6230 "number_recall": {6231 "mean": 0.9935714285714285,6232 "ci_low": 0.981107142857143,6233 "ci_high": 1.0,6234 "n_items": 70,6235 "n_groups": 96236 },6237 "placeholders_exact": {6238 "mean": 0.9428571428571428,6239 "ci_low": 0.8378378378378378,6240 "ci_high": 1.0,6241 "n_items": 70,6242 "n_groups": 96243 }6244 },6245 "related_work": {6246 "n": 6,6247 "chrf": {6248 "mean": 73.92872449883906,6249 "ci_low": 73.92872449883906,6250 "ci_high": 73.92872449883906,6251 "n_items": 6,6252 "n_groups": 16253 },6254 "edit_sim": {6255 "mean": 0.7173562600752282,6256 "ci_low": 0.7173562600752282,6257 "ci_high": 0.7173562600752282,6258 "n_items": 6,6259 "n_groups": 16260 },6261 "rougeL": {6262 "mean": 0.7266979219328021,6263 "ci_low": 0.7266979219328021,6264 "ci_high": 0.7266979219328021,6265 "n_items": 6,6266 "n_groups": 16267 },6268 "exact_match": {6269 "mean": 0.0,6270 "ci_low": 0.0,6271 "ci_high": 0.0,6272 "n_items": 6,6273 "n_groups": 16274 },6275 "number_recall": {6276 "mean": 1.0,6277 "ci_low": 1.0,6278 "ci_high": 1.0,6279 "n_items": 6,6280 "n_groups": 16281 },6282 "placeholders_exact": {6283 "mean": 0.8333333333333334,6284 "ci_low": 0.8333333333333334,6285 "ci_high": 0.8333333333333334,6286 "n_items": 6,6287 "n_groups": 16288 }6289 },6290 "results": {6291 "n": 78,6292 "chrf": {6293 "mean": 68.82346393108391,6294 "ci_low": 67.662601861647,6295 "ci_high": 70.13031747285414,6296 "n_items": 78,6297 "n_groups": 96298 },6299 "edit_sim": {6300 "mean": 0.6332863699851626,6301 "ci_low": 0.6007091338663518,6302 "ci_high": 0.6620933164326948,6303 "n_items": 78,6304 "n_groups": 96305 },6306 "rougeL": {6307 "mean": 0.6576145448288634,6308 "ci_low": 0.6368505682399132,6309 "ci_high": 0.6760361800139354,6310 "n_items": 78,6311 "n_groups": 96312 },6313 "exact_match": {6314 "mean": 0.0,6315 "ci_low": 0.0,6316 "ci_high": 0.0,6317 "n_items": 78,6318 "n_groups": 96319 },6320 "number_recall": {6321 "mean": 0.9952686202686203,6322 "ci_low": 0.9909988385598142,6323 "ci_high": 1.0,6324 "n_items": 78,6325 "n_groups": 96326 },6327 "placeholders_exact": {6328 "mean": 1.0,6329 "ci_low": 1.0,6330 "ci_high": 1.0,6331 "n_items": 78,6332 "n_groups": 96333 }6334 },6335 "theory": {6336 "n": 59,6337 "chrf": {6338 "mean": 71.11251089841811,6339 "ci_low": 70.09045998467492,6340 "ci_high": 73.09705949994621,6341 "n_items": 59,6342 "n_groups": 66343 },6344 "edit_sim": {6345 "mean": 0.682534721914688,6346 "ci_low": 0.664577587214411,6347 "ci_high": 0.7011092639753812,6348 "n_items": 59,6349 "n_groups": 66350 },6351 "rougeL": {6352 "mean": 0.6905792385574508,6353 "ci_low": 0.6810525406414305,6354 "ci_high": 0.7064019769094555,6355 "n_items": 59,6356 "n_groups": 66357 },6358 "exact_match": {6359 "mean": 0.0,6360 "ci_low": 0.0,6361 "ci_high": 0.0,6362 "n_items": 59,6363 "n_groups": 66364 },6365 "number_recall": {6366 "mean": 0.970741123283496,6367 "ci_low": 0.9404761904761904,6368 "ci_high": 0.9952380952380953,6369 "n_items": 59,6370 "n_groups": 66371 },6372 "placeholders_exact": {6373 "mean": 1.0,6374 "ci_low": 1.0,6375 "ci_high": 1.0,6376 "n_items": 59,6377 "n_groups": 66378 }6379 }6380 }6381 },6382 "htr-m3tokenonly-tiny": {6383 "overall": {6384 "n": 812,6385 "chrf": {6386 "mean": 70.51704299784237,6387 "ci_low": 69.66304697439634,6388 "ci_high": 71.51605231837223,6389 "n_items": 812,6390 "n_groups": 546391 },6392 "edit_sim": {6393 "mean": 0.6676779108890766,6394 "ci_low": 0.656541199647137,6395 "ci_high": 0.6786951322267186,6396 "n_items": 812,6397 "n_groups": 546398 },6399 "rougeL": {6400 "mean": 0.6801984873644122,6401 "ci_low": 0.6693686855318256,6402 "ci_high": 0.6918751915987553,6403 "n_items": 812,6404 "n_groups": 546405 },6406 "exact_match": {6407 "mean": 0.0,6408 "ci_low": 0.0,6409 "ci_high": 0.0,6410 "n_items": 812,6411 "n_groups": 546412 },6413 "number_recall": {6414 "mean": 0.9932313687450451,6415 "ci_low": 0.9884923571120182,6416 "ci_high": 0.9967426812239194,6417 "n_items": 812,6418 "n_groups": 546419 },6420 "placeholders_exact": {6421 "mean": 0.9901477832512315,6422 "ci_low": 0.9696550677620218,6423 "ci_high": 1.0,6424 "n_items": 812,6425 "n_groups": 546426 }6427 },6428 "by_variant": {6429 "A": {6430 "n": 133,6431 "chrf": {6432 "mean": 69.28756740082747,6433 "ci_low": 68.3811838242303,6434 "ci_high": 70.2461712508295,6435 "n_items": 133,6436 "n_groups": 506437 },6438 "edit_sim": {6439 "mean": 0.6377190234629084,6440 "ci_low": 0.6249659944786086,6441 "ci_high": 0.6490781902767792,6442 "n_items": 133,6443 "n_groups": 506444 },6445 "rougeL": {6446 "mean": 0.650514249196063,6447 "ci_low": 0.6370268400872211,6448 "ci_high": 0.6637521294926461,6449 "n_items": 133,6450 "n_groups": 506451 },6452 "exact_match": {6453 "mean": 0.0,6454 "ci_low": 0.0,6455 "ci_high": 0.0,6456 "n_items": 133,6457 "n_groups": 506458 },6459 "number_recall": {6460 "mean": 0.9966594116871124,6461 "ci_low": 0.9935264064128088,6462 "ci_high": 0.9995462794918331,6463 "n_items": 133,6464 "n_groups": 506465 },6466 "placeholders_exact": {6467 "mean": 0.9924812030075187,6468 "ci_low": 0.9735084686163042,6469 "ci_high": 1.0,6470 "n_items": 133,6471 "n_groups": 506472 }6473 },6474 "B": {6475 "n": 136,6476 "chrf": {6477 "mean": 75.76931154549527,6478 "ci_low": 74.6946434602132,6479 "ci_high": 76.98699469394428,6480 "n_items": 136,6481 "n_groups": 526482 },6483 "edit_sim": {6484 "mean": 0.7460488878303748,6485 "ci_low": 0.7328182040281798,6486 "ci_high": 0.7579720121467328,6487 "n_items": 136,6488 "n_groups": 526489 },6490 "rougeL": {6491 "mean": 0.7520358946543761,6492 "ci_low": 0.7391181140605722,6493 "ci_high": 0.7660264225920401,6494 "n_items": 136,6495 "n_groups": 526496 },6497 "exact_match": {6498 "mean": 0.0,6499 "ci_low": 0.0,6500 "ci_high": 0.0,6501 "n_items": 136,6502 "n_groups": 526503 },6504 "number_recall": {6505 "mean": 0.989753377427526,6506 "ci_low": 0.9776147414976489,6507 "ci_high": 0.9973516385497443,6508 "n_items": 136,6509 "n_groups": 526510 },6511 "placeholders_exact": {6512 "mean": 0.9926470588235294,6513 "ci_low": 0.9734513274336283,6514 "ci_high": 1.0,6515 "n_items": 136,6516 "n_groups": 526517 }6518 },6519 "C": {6520 "n": 135,6521 "chrf": {6522 "mean": 72.20855430761509,6523 "ci_low": 71.02034135784082,6524 "ci_high": 73.65936771442483,6525 "n_items": 135,6526 "n_groups": 536527 },6528 "edit_sim": {6529 "mean": 0.7202975266589831,6530 "ci_low": 0.7079435052390073,6531 "ci_high": 0.734012375864158,6532 "n_items": 135,6533 "n_groups": 536534 },6535 "rougeL": {6536 "mean": 0.7383173319740512,6537 "ci_low": 0.7249848497696032,6538 "ci_high": 0.7547976277485345,6539 "n_items": 135,6540 "n_groups": 536541 },6542 "exact_match": {6543 "mean": 0.0,6544 "ci_low": 0.0,6545 "ci_high": 0.0,6546 "n_items": 135,6547 "n_groups": 536548 },6549 "number_recall": {6550 "mean": 0.9972700691121744,6551 "ci_low": 0.9939927320294356,6552 "ci_high": 0.9995825319111733,6553 "n_items": 135,6554 "n_groups": 536555 },6556 "placeholders_exact": {6557 "mean": 0.9925925925925926,6558 "ci_low": 0.974570838765754,6559 "ci_high": 1.0,6560 "n_items": 135,6561 "n_groups": 536562 }6563 },6564 "D": {6565 "n": 136,6566 "chrf": {6567 "mean": 67.78298986269641,6568 "ci_low": 66.7678806848577,6569 "ci_high": 68.80957908492796,6570 "n_items": 136,6571 "n_groups": 516572 },6573 "edit_sim": {6574 "mean": 0.6165457891136148,6575 "ci_low": 0.6019967565606129,6576 "ci_high": 0.632038022351286,6577 "n_items": 136,6578 "n_groups": 516579 },6580 "rougeL": {6581 "mean": 0.6316173883814731,6582 "ci_low": 0.6187984374735611,6583 "ci_high": 0.6448207426648707,6584 "n_items": 136,6585 "n_groups": 516586 },6587 "exact_match": {6588 "mean": 0.0,6589 "ci_low": 0.0,6590 "ci_high": 0.0,6591 "n_items": 136,6592 "n_groups": 516593 },6594 "number_recall": {6595 "mean": 0.9871827658495617,6596 "ci_low": 0.9743724460034767,6597 "ci_high": 0.9957075508362964,6598 "n_items": 136,6599 "n_groups": 516600 },6601 "placeholders_exact": {6602 "mean": 0.9852941176470589,6603 "ci_low": 0.96,6604 "ci_high": 1.0,6605 "n_items": 136,6606 "n_groups": 516607 }6608 },6609 "E": {6610 "n": 272,6611 "chrf": {6612 "mean": 69.0195758341759,6613 "ci_low": 68.05299710234642,6614 "ci_high": 70.11883663299167,6615 "n_items": 272,6616 "n_groups": 526617 },6618 "edit_sim": {6619 "mean": 0.6425911446985959,6620 "ci_low": 0.6291786931253331,6621 "ci_high": 0.6578865437769379,6622 "n_items": 272,6623 "n_groups": 526624 },6625 "rougeL": {6626 "mean": 0.6542393025274043,6627 "ci_low": 0.6405038273329637,6628 "ci_high": 0.6689946589916344,6629 "n_items": 272,6630 "n_groups": 526631 },6632 "exact_match": {6633 "mean": 0.0,6634 "ci_low": 0.0,6635 "ci_high": 0.0,6636 "n_items": 272,6637 "n_groups": 526638 },6639 "number_recall": {6640 "mean": 0.9943139516572179,6641 "ci_low": 0.9893469043433689,6642 "ci_high": 0.9978532068799143,6643 "n_items": 272,6644 "n_groups": 526645 },6646 "placeholders_exact": {6647 "mean": 0.9889705882352942,6648 "ci_low": 0.968623476417594,6649 "ci_high": 1.0,6650 "n_items": 272,6651 "n_groups": 526652 }6653 }6654 },6655 "by_discipline": {6656 "biomedicine": {6657 "n": 380,6658 "chrf": {6659 "mean": 70.9706038884964,6660 "ci_low": 69.84878521280112,6661 "ci_high": 72.05374790812948,6662 "n_items": 380,6663 "n_groups": 316664 },6665 "edit_sim": {6666 "mean": 0.6633249230184879,6667 "ci_low": 0.6467943890132452,6668 "ci_high": 0.6747436414788491,6669 "n_items": 380,6670 "n_groups": 316671 },6672 "rougeL": {6673 "mean": 0.6807409527522099,6674 "ci_low": 0.6654298219289178,6675 "ci_high": 0.6914034633101276,6676 "n_items": 380,6677 "n_groups": 316678 },6679 "exact_match": {6680 "mean": 0.0,6681 "ci_low": 0.0,6682 "ci_high": 0.0,6683 "n_items": 380,6684 "n_groups": 316685 },6686 "number_recall": {6687 "mean": 0.9961468363892186,6688 "ci_low": 0.9917249719915855,6689 "ci_high": 0.9991351341502813,6690 "n_items": 380,6691 "n_groups": 316692 },6693 "placeholders_exact": {6694 "mean": 0.9947368421052631,6695 "ci_low": 0.9817069682424513,6696 "ci_high": 1.0,6697 "n_items": 380,6698 "n_groups": 316699 }6700 },6701 "engineering": {6702 "n": 12,6703 "chrf": {6704 "mean": 73.6097830375017,6705 "ci_low": 73.33121511828375,6706 "ci_high": 73.88835095671966,6707 "n_items": 12,6708 "n_groups": 26709 },6710 "edit_sim": {6711 "mean": 0.7109433521430701,6712 "ci_low": 0.7072362529106216,6713 "ci_high": 0.7146504513755189,6714 "n_items": 12,6715 "n_groups": 26716 },6717 "rougeL": {6718 "mean": 0.7039454948061178,6719 "ci_low": 0.6970101912176924,6720 "ci_high": 0.710880798394543,6721 "n_items": 12,6722 "n_groups": 26723 },6724 "exact_match": {6725 "mean": 0.0,6726 "ci_low": 0.0,6727 "ci_high": 0.0,6728 "n_items": 12,6729 "n_groups": 26730 },6731 "number_recall": {6732 "mean": 1.0,6733 "ci_low": 1.0,6734 "ci_high": 1.0,6735 "n_items": 12,6736 "n_groups": 26737 },6738 "placeholders_exact": {6739 "mean": 1.0,6740 "ci_low": 1.0,6741 "ci_high": 1.0,6742 "n_items": 12,6743 "n_groups": 26744 }6745 },6746 "finance": {6747 "n": 96,6748 "chrf": {6749 "mean": 68.38075537520261,6750 "ci_low": 68.38075537520261,6751 "ci_high": 68.38075537520261,6752 "n_items": 96,6753 "n_groups": 16754 },6755 "edit_sim": {6756 "mean": 0.6576105505957061,6757 "ci_low": 0.6576105505957061,6758 "ci_high": 0.6576105505957061,6759 "n_items": 96,6760 "n_groups": 16761 },6762 "rougeL": {6763 "mean": 0.6534850152010514,6764 "ci_low": 0.6534850152010514,6765 "ci_high": 0.6534850152010514,6766 "n_items": 96,6767 "n_groups": 16768 },6769 "exact_match": {6770 "mean": 0.0,6771 "ci_low": 0.0,6772 "ci_high": 0.0,6773 "n_items": 96,6774 "n_groups": 16775 },6776 "number_recall": {6777 "mean": 0.9965277777777777,6778 "ci_low": 0.9965277777777777,6779 "ci_high": 0.9965277777777777,6780 "n_items": 96,6781 "n_groups": 16782 },6783 "placeholders_exact": {6784 "mean": 1.0,6785 "ci_low": 1.0,6786 "ci_high": 1.0,6787 "n_items": 96,6788 "n_groups": 16789 }6790 },6791 "life_sciences": {6792 "n": 162,6793 "chrf": {6794 "mean": 71.51567762838633,6795 "ci_low": 69.34395252911627,6796 "ci_high": 73.93975896642823,6797 "n_items": 162,6798 "n_groups": 156799 },6800 "edit_sim": {6801 "mean": 0.6787806459578406,6802 "ci_low": 0.6587114263244175,6803 "ci_high": 0.7033473936143861,6804 "n_items": 162,6805 "n_groups": 156806 },6807 "rougeL": {6808 "mean": 0.6959274809847258,6809 "ci_low": 0.6723184931300146,6810 "ci_high": 0.7238767666603599,6811 "n_items": 162,6812 "n_groups": 156813 },6814 "exact_match": {6815 "mean": 0.0,6816 "ci_low": 0.0,6817 "ci_high": 0.0,6818 "n_items": 162,6819 "n_groups": 156820 },6821 "number_recall": {6822 "mean": 0.9874572444016889,6823 "ci_low": 0.9754914800599818,6824 "ci_high": 0.996887611307222,6825 "n_items": 162,6826 "n_groups": 156827 },6828 "placeholders_exact": {6829 "mean": 0.9629629629629629,6830 "ci_low": 0.8758620689655172,6831 "ci_high": 1.0,6832 "n_items": 162,6833 "n_groups": 156834 }6835 },6836 "mathematics": {6837 "n": 60,6838 "chrf": {6839 "mean": 69.75470934079223,6840 "ci_low": 69.75470934079223,6841 "ci_high": 69.75470934079223,6842 "n_items": 60,6843 "n_groups": 16844 },6845 "edit_sim": {6846 "mean": 0.700069265182059,6847 "ci_low": 0.700069265182059,6848 "ci_high": 0.700069265182059,6849 "n_items": 60,6850 "n_groups": 16851 },6852 "rougeL": {6853 "mean": 0.6896043878395727,6854 "ci_low": 0.6896043878395727,6855 "ci_high": 0.6896043878395727,6856 "n_items": 60,6857 "n_groups": 16858 },6859 "exact_match": {6860 "mean": 0.0,6861 "ci_low": 0.0,6862 "ci_high": 0.0,6863 "n_items": 60,6864 "n_groups": 16865 },6866 "number_recall": {6867 "mean": 1.0,6868 "ci_low": 1.0,6869 "ci_high": 1.0,6870 "n_items": 60,6871 "n_groups": 16872 },6873 "placeholders_exact": {6874 "mean": 1.0,6875 "ci_low": 1.0,6876 "ci_high": 1.0,6877 "n_items": 60,6878 "n_groups": 16879 }6880 },6881 "social_sciences": {6882 "n": 30,6883 "chrf": {6884 "mean": 69.43622939404847,6885 "ci_low": 56.91623386315727,6886 "ci_high": 72.56622827677127,6887 "n_items": 30,6888 "n_groups": 26889 },6890 "edit_sim": {6891 "mean": 0.6964297916111875,6892 "ci_low": 0.5890375841482238,6893 "ci_high": 0.7232778434769284,6894 "n_items": 30,6895 "n_groups": 26896 },6897 "rougeL": {6898 "mean": 0.6848959138073802,6899 "ci_low": 0.4941061286002851,6900 "ci_high": 0.732593360109154,6901 "n_items": 30,6902 "n_groups": 26903 },6904 "exact_match": {6905 "mean": 0.0,6906 "ci_low": 0.0,6907 "ci_high": 0.0,6908 "n_items": 30,6909 "n_groups": 26910 },6911 "number_recall": {6912 "mean": 0.961111111111111,6913 "ci_low": 0.951388888888889,6914 "ci_high": 1.0,6915 "n_items": 30,6916 "n_groups": 26917 },6918 "placeholders_exact": {6919 "mean": 1.0,6920 "ci_low": 1.0,6921 "ci_high": 1.0,6922 "n_items": 30,6923 "n_groups": 26924 }6925 },6926 "statistics": {6927 "n": 72,6928 "chrf": {6929 "mean": 69.29486536225457,6930 "ci_low": 67.36116724323226,6931 "ci_high": 70.26171442176572,6932 "n_items": 72,6933 "n_groups": 26934 },6935 "edit_sim": {6936 "mean": 0.6329103542718166,6937 "ci_low": 0.6143442228798842,6938 "ci_high": 0.6421934199677829,6939 "n_items": 72,6940 "n_groups": 26941 },6942 "rougeL": {6943 "mean": 0.6638098568467679,6944 "ci_low": 0.6618202224008195,6945 "ci_high": 0.664804674069742,6946 "n_items": 72,6947 "n_groups": 26948 },6949 "exact_match": {6950 "mean": 0.0,6951 "ci_low": 0.0,6952 "ci_high": 0.0,6953 "n_items": 72,6954 "n_groups": 26955 },6956 "number_recall": {6957 "mean": 0.9930555555555556,6958 "ci_low": 0.9895833333333334,6959 "ci_high": 1.0,6960 "n_items": 72,6961 "n_groups": 26962 },6963 "placeholders_exact": {6964 "mean": 1.0,6965 "ci_low": 1.0,6966 "ci_high": 1.0,6967 "n_items": 72,6968 "n_groups": 26969 }6970 }6971 },6972 "by_era": {6973 "1980-1999": {6974 "n": 8,6975 "chrf": {6976 "mean": 71.00296465196948,6977 "ci_low": 70.13976370662003,6978 "ci_high": 72.44163289421853,6979 "n_items": 8,6980 "n_groups": 26981 },6982 "edit_sim": {6983 "mean": 0.688110482760786,6984 "ci_low": 0.6657344816683588,6985 "ci_high": 0.7254038179148311,6986 "n_items": 8,6987 "n_groups": 26988 },6989 "rougeL": {6990 "mean": 0.6859815029006524,6991 "ci_low": 0.6622600740288845,6992 "ci_high": 0.7255172176869323,6993 "n_items": 8,6994 "n_groups": 26995 },6996 "exact_match": {6997 "mean": 0.0,6998 "ci_low": 0.0,6999 "ci_high": 0.0,7000 "n_items": 8,7001 "n_groups": 27002 },7003 "number_recall": {7004 "mean": 1.0,7005 "ci_low": 1.0,7006 "ci_high": 1.0,7007 "n_items": 8,7008 "n_groups": 27009 },7010 "placeholders_exact": {7011 "mean": 1.0,7012 "ci_low": 1.0,7013 "ci_high": 1.0,7014 "n_items": 8,7015 "n_groups": 27016 }7017 },7018 "2000-2009": {7019 "n": 89,7020 "chrf": {7021 "mean": 69.51157018488954,7022 "ci_low": 67.33041535684627,7023 "ci_high": 73.51174025266579,7024 "n_items": 89,7025 "n_groups": 77026 },7027 "edit_sim": {7028 "mean": 0.6560766136135404,7029 "ci_low": 0.6293626108918497,7030 "ci_high": 0.7055678414823837,7031 "n_items": 89,7032 "n_groups": 77033 },7034 "rougeL": {7035 "mean": 0.6662665324379763,7036 "ci_low": 0.6383524984739539,7037 "ci_high": 0.7153671577226693,7038 "n_items": 89,7039 "n_groups": 77040 },7041 "exact_match": {7042 "mean": 0.0,7043 "ci_low": 0.0,7044 "ci_high": 0.0,7045 "n_items": 89,7046 "n_groups": 77047 },7048 "number_recall": {7049 "mean": 0.9925093632958801,7050 "ci_low": 0.977266196447231,7051 "ci_high": 1.0,7052 "n_items": 89,7053 "n_groups": 77054 },7055 "placeholders_exact": {7056 "mean": 1.0,7057 "ci_low": 1.0,7058 "ci_high": 1.0,7059 "n_items": 89,7060 "n_groups": 77061 }7062 },7063 "2010-2017": {7064 "n": 715,7065 "chrf": {7066 "mean": 70.63676286793996,7067 "ci_low": 69.68875273587115,7068 "ci_high": 71.70181154034354,7069 "n_items": 715,7070 "n_groups": 457071 },7072 "edit_sim": {7073 "mean": 0.6688933722632711,7074 "ci_low": 0.6575285621737268,7075 "ci_high": 0.6799016783756333,7076 "n_items": 715,7077 "n_groups": 457078 },7079 "rougeL": {7080 "mean": 0.6818679696919129,7081 "ci_low": 0.6694633655345879,7082 "ci_high": 0.6938297598121401,7083 "n_items": 715,7084 "n_groups": 457085 },7086 "exact_match": {7087 "mean": 0.0,7088 "ci_low": 0.0,7089 "ci_high": 0.0,7090 "n_items": 715,7091 "n_groups": 457092 },7093 "number_recall": {7094 "mean": 0.9932455078148857,7095 "ci_low": 0.9880518439288034,7096 "ci_high": 0.9968422717432697,7097 "n_items": 715,7098 "n_groups": 457099 },7100 "placeholders_exact": {7101 "mean": 0.9888111888111888,7102 "ci_low": 0.9662566323992704,7103 "ci_high": 1.0,7104 "n_items": 715,7105 "n_groups": 457106 }7107 }7108 },7109 "by_role": {7110 "abstract": {7111 "n": 83,7112 "chrf": {7113 "mean": 73.29205117384657,7114 "ci_low": 71.54407762706099,7115 "ci_high": 74.91609816171844,7116 "n_items": 83,7117 "n_groups": 157118 },7119 "edit_sim": {7120 "mean": 0.6944532956836769,7121 "ci_low": 0.6708487803221332,7122 "ci_high": 0.7190677518633595,7123 "n_items": 83,7124 "n_groups": 157125 },7126 "rougeL": {7127 "mean": 0.7065440051468485,7128 "ci_low": 0.6781562427364521,7129 "ci_high": 0.7340301512751376,7130 "n_items": 83,7131 "n_groups": 157132 },7133 "exact_match": {7134 "mean": 0.0,7135 "ci_low": 0.0,7136 "ci_high": 0.0,7137 "n_items": 83,7138 "n_groups": 157139 },7140 "number_recall": {7141 "mean": 0.9939759036144579,7142 "ci_low": 0.98125,7143 "ci_high": 1.0,7144 "n_items": 83,7145 "n_groups": 157146 },7147 "placeholders_exact": {7148 "mean": 1.0,7149 "ci_low": 1.0,7150 "ci_high": 1.0,7151 "n_items": 83,7152 "n_groups": 157153 }7154 },7155 "background": {7156 "n": 58,7157 "chrf": {7158 "mean": 71.05102231952263,7159 "ci_low": 67.94558509660719,7160 "ci_high": 74.20897942485257,7161 "n_items": 58,7162 "n_groups": 77163 },7164 "edit_sim": {7165 "mean": 0.6831545505377362,7166 "ci_low": 0.6517389701948268,7167 "ci_high": 0.7156696058469023,7168 "n_items": 58,7169 "n_groups": 77170 },7171 "rougeL": {7172 "mean": 0.6939903294896314,7173 "ci_low": 0.6647521712193254,7174 "ci_high": 0.7214925358940429,7175 "n_items": 58,7176 "n_groups": 77177 },7178 "exact_match": {7179 "mean": 0.0,7180 "ci_low": 0.0,7181 "ci_high": 0.0,7182 "n_items": 58,7183 "n_groups": 77184 },7185 "number_recall": {7186 "mean": 0.9875478927203064,7187 "ci_low": 0.9668458781362007,7188 "ci_high": 1.0,7189 "n_items": 58,7190 "n_groups": 77191 },7192 "placeholders_exact": {7193 "mean": 1.0,7194 "ci_low": 1.0,7195 "ci_high": 1.0,7196 "n_items": 58,7197 "n_groups": 77198 }7199 },7200 "conclusion": {7201 "n": 47,7202 "chrf": {7203 "mean": 69.56046028114898,7204 "ci_low": 67.19148378943422,7205 "ci_high": 71.70186527065503,7206 "n_items": 47,7207 "n_groups": 87208 },7209 "edit_sim": {7210 "mean": 0.6741810934714599,7211 "ci_low": 0.6466942106448325,7212 "ci_high": 0.7017683462145508,7213 "n_items": 47,7214 "n_groups": 87215 },7216 "rougeL": {7217 "mean": 0.6744862932893998,7218 "ci_low": 0.6474361509005748,7219 "ci_high": 0.6963933059876064,7220 "n_items": 47,7221 "n_groups": 87222 },7223 "exact_match": {7224 "mean": 0.0,7225 "ci_low": 0.0,7226 "ci_high": 0.0,7227 "n_items": 47,7228 "n_groups": 87229 },7230 "number_recall": {7231 "mean": 1.0,7232 "ci_low": 1.0,7233 "ci_high": 1.0,7234 "n_items": 47,7235 "n_groups": 87236 },7237 "placeholders_exact": {7238 "mean": 1.0,7239 "ci_low": 1.0,7240 "ci_high": 1.0,7241 "n_items": 47,7242 "n_groups": 87243 }7244 },7245 "data": {7246 "n": 69,7247 "chrf": {7248 "mean": 72.05960071551819,7249 "ci_low": 69.70093126880197,7250 "ci_high": 74.88457952985594,7251 "n_items": 69,7252 "n_groups": 107253 },7254 "edit_sim": {7255 "mean": 0.6394556499405248,7256 "ci_low": 0.5883644845697493,7257 "ci_high": 0.6887840085969775,7258 "n_items": 69,7259 "n_groups": 107260 },7261 "rougeL": {7262 "mean": 0.6847769844505366,7263 "ci_low": 0.6443718689825831,7264 "ci_high": 0.7283484502046395,7265 "n_items": 69,7266 "n_groups": 107267 },7268 "exact_match": {7269 "mean": 0.0,7270 "ci_low": 0.0,7271 "ci_high": 0.0,7272 "n_items": 69,7273 "n_groups": 107274 },7275 "number_recall": {7276 "mean": 0.9863653699466057,7277 "ci_low": 0.9641917293233082,7278 "ci_high": 1.0,7279 "n_items": 69,7280 "n_groups": 107281 },7282 "placeholders_exact": {7283 "mean": 0.9710144927536232,7284 "ci_low": 0.9015983606557377,7285 "ci_high": 1.0,7286 "n_items": 69,7287 "n_groups": 107288 }7289 },7290 "discussion": {7291 "n": 103,7292 "chrf": {7293 "mean": 70.4733800791501,7294 "ci_low": 68.7178068962177,7295 "ci_high": 72.35322646440481,7296 "n_items": 103,7297 "n_groups": 147298 },7299 "edit_sim": {7300 "mean": 0.6839723588608162,7301 "ci_low": 0.662670756170262,7302 "ci_high": 0.7071936331205854,7303 "n_items": 103,7304 "n_groups": 147305 },7306 "rougeL": {7307 "mean": 0.6899686428396691,7308 "ci_low": 0.6624921819562256,7309 "ci_high": 0.7180228198881748,7310 "n_items": 103,7311 "n_groups": 147312 },7313 "exact_match": {7314 "mean": 0.0,7315 "ci_low": 0.0,7316 "ci_high": 0.0,7317 "n_items": 103,7318 "n_groups": 147319 },7320 "number_recall": {7321 "mean": 0.9911003236245954,7322 "ci_low": 0.9775936303991251,7323 "ci_high": 1.0,7324 "n_items": 103,7325 "n_groups": 147326 },7327 "placeholders_exact": {7328 "mean": 1.0,7329 "ci_low": 1.0,7330 "ci_high": 1.0,7331 "n_items": 103,7332 "n_groups": 147333 }7334 },7335 "introduction": {7336 "n": 99,7337 "chrf": {7338 "mean": 69.54010347501035,7339 "ci_low": 67.23319231445988,7340 "ci_high": 71.57244440597238,7341 "n_items": 99,7342 "n_groups": 147343 },7344 "edit_sim": {7345 "mean": 0.6535854271027549,7346 "ci_low": 0.6208124562504518,7347 "ci_high": 0.6830583183841363,7348 "n_items": 99,7349 "n_groups": 147350 },7351 "rougeL": {7352 "mean": 0.6635648838074212,7353 "ci_low": 0.6393098236418623,7354 "ci_high": 0.6865952761886632,7355 "n_items": 99,7356 "n_groups": 147357 },7358 "exact_match": {7359 "mean": 0.0,7360 "ci_low": 0.0,7361 "ci_high": 0.0,7362 "n_items": 99,7363 "n_groups": 147364 },7365 "number_recall": {7366 "mean": 0.9924242424242424,7367 "ci_low": 0.9798850574712644,7368 "ci_high": 1.0,7369 "n_items": 99,7370 "n_groups": 147371 },7372 "placeholders_exact": {7373 "mean": 1.0,7374 "ci_low": 1.0,7375 "ci_high": 1.0,7376 "n_items": 99,7377 "n_groups": 147378 }7379 },7380 "limitations": {7381 "n": 42,7382 "chrf": {7383 "mean": 69.98362478392846,7384 "ci_low": 68.54679803478157,7385 "ci_high": 71.59318078497365,7386 "n_items": 42,7387 "n_groups": 47388 },7389 "edit_sim": {7390 "mean": 0.6617790381032176,7391 "ci_low": 0.6291337987652681,7392 "ci_high": 0.6851893916149331,7393 "n_items": 42,7394 "n_groups": 47395 },7396 "rougeL": {7397 "mean": 0.6645824092722856,7398 "ci_low": 0.644425581035464,7399 "ci_high": 0.679700030449902,7400 "n_items": 42,7401 "n_groups": 47402 },7403 "exact_match": {7404 "mean": 0.0,7405 "ci_low": 0.0,7406 "ci_high": 0.0,7407 "n_items": 42,7408 "n_groups": 47409 },7410 "number_recall": {7411 "mean": 1.0,7412 "ci_low": 1.0,7413 "ci_high": 1.0,7414 "n_items": 42,7415 "n_groups": 47416 },7417 "placeholders_exact": {7418 "mean": 1.0,7419 "ci_low": 1.0,7420 "ci_high": 1.0,7421 "n_items": 42,7422 "n_groups": 47423 }7424 },7425 "methods": {7426 "n": 98,7427 "chrf": {7428 "mean": 70.77076031687383,7429 "ci_low": 67.31516830096255,7430 "ci_high": 74.7831177346705,7431 "n_items": 98,7432 "n_groups": 127433 },7434 "edit_sim": {7435 "mean": 0.6698236184309068,7436 "ci_low": 0.6462039058668613,7437 "ci_high": 0.6953537818105927,7438 "n_items": 98,7439 "n_groups": 127440 },7441 "rougeL": {7442 "mean": 0.6829198243385478,7443 "ci_low": 0.6559305205362068,7444 "ci_high": 0.7106508487630141,7445 "n_items": 98,7446 "n_groups": 127447 },7448 "exact_match": {7449 "mean": 0.0,7450 "ci_low": 0.0,7451 "ci_high": 0.0,7452 "n_items": 98,7453 "n_groups": 127454 },7455 "number_recall": {7456 "mean": 0.9965986394557823,7457 "ci_low": 0.9883720930232558,7458 "ci_high": 1.0,7459 "n_items": 98,7460 "n_groups": 127461 },7462 "placeholders_exact": {7463 "mean": 1.0,7464 "ci_low": 1.0,7465 "ci_high": 1.0,7466 "n_items": 98,7467 "n_groups": 127468 }7469 },7470 "other": {7471 "n": 70,7472 "chrf": {7473 "mean": 69.07521943448673,7474 "ci_low": 64.60069620130095,7475 "ci_high": 72.65374637937191,7476 "n_items": 70,7477 "n_groups": 97478 },7479 "edit_sim": {7480 "mean": 0.6641013931128715,7481 "ci_low": 0.6271707322371064,7482 "ci_high": 0.6961206007387606,7483 "n_items": 70,7484 "n_groups": 97485 },7486 "rougeL": {7487 "mean": 0.6599365836408937,7488 "ci_low": 0.6068810564476687,7489 "ci_high": 0.705018685398762,7490 "n_items": 70,7491 "n_groups": 97492 },7493 "exact_match": {7494 "mean": 0.0,7495 "ci_low": 0.0,7496 "ci_high": 0.0,7497 "n_items": 70,7498 "n_groups": 97499 },7500 "number_recall": {7501 "mean": 0.9971428571428571,7502 "ci_low": 0.9894736842105263,7503 "ci_high": 1.0,7504 "n_items": 70,7505 "n_groups": 97506 },7507 "placeholders_exact": {7508 "mean": 0.9142857142857143,7509 "ci_low": 0.7567567567567568,7510 "ci_high": 1.0,7511 "n_items": 70,7512 "n_groups": 97513 }7514 },7515 "related_work": {7516 "n": 6,7517 "chrf": {7518 "mean": 73.33121511828375,7519 "ci_low": 73.33121511828375,7520 "ci_high": 73.33121511828375,7521 "n_items": 6,7522 "n_groups": 17523 },7524 "edit_sim": {7525 "mean": 0.7072362529106214,7526 "ci_low": 0.7072362529106214,7527 "ci_high": 0.7072362529106214,7528 "n_items": 6,7529 "n_groups": 17530 },7531 "rougeL": {7532 "mean": 0.710880798394543,7533 "ci_low": 0.710880798394543,7534 "ci_high": 0.710880798394543,7535 "n_items": 6,7536 "n_groups": 17537 },7538 "exact_match": {7539 "mean": 0.0,7540 "ci_low": 0.0,7541 "ci_high": 0.0,7542 "n_items": 6,7543 "n_groups": 17544 },7545 "number_recall": {7546 "mean": 1.0,7547 "ci_low": 1.0,7548 "ci_high": 1.0,7549 "n_items": 6,7550 "n_groups": 17551 },7552 "placeholders_exact": {7553 "mean": 1.0,7554 "ci_low": 1.0,7555 "ci_high": 1.0,7556 "n_items": 6,7557 "n_groups": 17558 }7559 },7560 "results": {7561 "n": 78,7562 "chrf": {7563 "mean": 68.38502138878077,7564 "ci_low": 67.22245247858227,7565 "ci_high": 69.68374335357333,7566 "n_items": 78,7567 "n_groups": 97568 },7569 "edit_sim": {7570 "mean": 0.6327333992710802,7571 "ci_low": 0.5991298751767979,7572 "ci_high": 0.6644913844045673,7573 "n_items": 78,7574 "n_groups": 97575 },7576 "rougeL": {7577 "mean": 0.6598909540015837,7578 "ci_low": 0.6359150308604006,7579 "ci_high": 0.6814615334903775,7580 "n_items": 78,7581 "n_groups": 97582 },7583 "exact_match": {7584 "mean": 0.0,7585 "ci_low": 0.0,7586 "ci_high": 0.0,7587 "n_items": 78,7588 "n_groups": 97589 },7590 "number_recall": {7591 "mean": 0.993131868131868,7592 "ci_low": 0.9857690202517788,7593 "ci_high": 1.0,7594 "n_items": 78,7595 "n_groups": 97596 },7597 "placeholders_exact": {7598 "mean": 1.0,7599 "ci_low": 1.0,7600 "ci_high": 1.0,7601 "n_items": 78,7602 "n_groups": 97603 }7604 },7605 "theory": {7606 "n": 59,7607 "chrf": {7608 "mean": 70.96314959484538,7609 "ci_low": 69.84892061727838,7610 "ci_high": 72.69302140433444,7611 "n_items": 59,7612 "n_groups": 67613 },7614 "edit_sim": {7615 "mean": 0.6848755982886725,7616 "ci_low": 0.6688827466980236,7617 "ci_high": 0.6997333632092217,7618 "n_items": 59,7619 "n_groups": 67620 },7621 "rougeL": {7622 "mean": 0.6939910974447333,7623 "ci_low": 0.6839585291448055,7624 "ci_high": 0.7058044121049469,7625 "n_items": 59,7626 "n_groups": 67627 },7628 "exact_match": {7629 "mean": 0.0,7630 "ci_low": 0.0,7631 "ci_high": 0.0,7632 "n_items": 59,7633 "n_groups": 67634 },7635 "number_recall": {7636 "mean": 0.9898745322474136,7637 "ci_low": 0.973570289359763,7638 "ci_high": 1.0,7639 "n_items": 59,7640 "n_groups": 67641 },7642 "placeholders_exact": {7643 "mean": 1.0,7644 "ci_low": 1.0,7645 "ci_high": 1.0,7646 "n_items": 59,7647 "n_groups": 67648 }7649 }7650 }7651 }7652 },7653 "paired_vs_baseline": {7654 "B1_generic_paraphrase": {7655 "chrf": {7656 "mean": -2.6722290052651734,7657 "ci_low": -3.052609202439086,7658 "ci_high": -2.3088503555661535,7659 "n_items": 812,7660 "n_groups": 54,7661 "p_value_two_sided": 0.0,7662 "p_holm": 0.0,7663 "significant_holm_0_05": true7664 },7665 "edit_sim": {7666 "mean": -0.09829727740280574,7667 "ci_low": -0.10464227746100756,7668 "ci_high": -0.09138247000078992,7669 "n_items": 812,7670 "n_groups": 54,7671 "p_value_two_sided": 0.0,7672 "p_holm": 0.0,7673 "significant_holm_0_05": true7674 },7675 "rougeL": {7676 "mean": -0.07403550648858839,7677 "ci_low": -0.07970098061554781,7678 "ci_high": -0.06843668976095463,7679 "n_items": 812,7680 "n_groups": 54,7681 "p_value_two_sided": 0.0,7682 "p_holm": 0.0,7683 "significant_holm_0_05": true7684 },7685 "exact_match": {7686 "mean": 0.0,7687 "ci_low": 0.0,7688 "ci_high": 0.0,7689 "n_items": 812,7690 "n_groups": 54,7691 "p_value_two_sided": 0.0,7692 "p_holm": 0.0,7693 "significant_holm_0_05": true7694 },7695 "number_recall": {7696 "mean": -0.0009236453201970443,7697 "ci_low": -0.002553916004540295,7698 "ci_high": 0.0,7699 "n_items": 812,7700 "n_groups": 54,7701 "p_value_two_sided": 0.0,7702 "p_holm": 0.0,7703 "significant_holm_0_05": true7704 },7705 "placeholders_exact": {7706 "mean": 0.0,7707 "ci_low": 0.0,7708 "ci_high": 0.0,7709 "n_items": 812,7710 "n_groups": 54,7711 "p_value_two_sided": 0.0,7712 "p_holm": 0.0,7713 "significant_holm_0_05": true7714 }7715 },7716 "htr-m1-small": {7717 "chrf": {7718 "mean": 0.3783281682954403,7719 "ci_low": -0.055305576680698146,7720 "ci_high": 0.7729166287382402,7721 "n_items": 812,7722 "n_groups": 54,7723 "p_value_two_sided": 0.0858,7724 "p_holm": 0.2574,7725 "significant_holm_0_05": false7726 },7727 "edit_sim": {7728 "mean": 0.0026507474451935844,7729 "ci_low": -0.0021961109922122974,7730 "ci_high": 0.007751443470956223,7731 "n_items": 812,7732 "n_groups": 54,7733 "p_value_two_sided": 0.296,7734 "p_holm": 0.296,7735 "significant_holm_0_05": false7736 },7737 "rougeL": {7738 "mean": 0.004711479813425156,7739 "ci_low": -0.001031054070365804,7740 "ci_high": 0.01002588383154493,7741 "n_items": 812,7742 "n_groups": 54,7743 "p_value_two_sided": 0.1012,7744 "p_holm": 0.2574,7745 "significant_holm_0_05": false7746 },7747 "exact_match": {7748 "mean": 0.0,7749 "ci_low": 0.0,7750 "ci_high": 0.0,7751 "n_items": 812,7752 "n_groups": 54,7753 "p_value_two_sided": 0.0,7754 "p_holm": 0.0,7755 "significant_holm_0_05": true7756 },7757 "number_recall": {7758 "mean": -0.009992680579438423,7759 "ci_low": -0.0181270918345258,7760 "ci_high": -0.004264141718819286,7761 "n_items": 812,7762 "n_groups": 54,7763 "p_value_two_sided": 0.0,7764 "p_holm": 0.0,7765 "significant_holm_0_05": true7766 },7767 "placeholders_exact": {7768 "mean": -0.006157635467980296,7769 "ci_low": -0.0136986301369863,7770 "ci_high": -0.0010857468892861437,7771 "n_items": 812,7772 "n_groups": 54,7773 "p_value_two_sided": 0.0,7774 "p_holm": 0.0,7775 "significant_holm_0_05": true7776 }7777 },7778 "htr-m1-tiny-s3": {7779 "chrf": {7780 "mean": 0.7231250972957879,7781 "ci_low": 0.2107740465273296,7782 "ci_high": 1.1077680302712047,7783 "n_items": 812,7784 "n_groups": 54,7785 "p_value_two_sided": 0.0082,7786 "p_holm": 0.0164,7787 "significant_holm_0_05": true7788 },7789 "edit_sim": {7790 "mean": 0.009067940955688834,7791 "ci_low": 0.003559066922009847,7792 "ci_high": 0.01448413209628965,7793 "n_items": 812,7794 "n_groups": 54,7795 "p_value_two_sided": 0.0004,7796 "p_holm": 0.0012000000000000001,7797 "significant_holm_0_05": true7798 },7799 "rougeL": {7800 "mean": 0.009609982494461726,7801 "ci_low": 0.0028611639508730837,7802 "ci_high": 0.014943615900588566,7803 "n_items": 812,7804 "n_groups": 54,7805 "p_value_two_sided": 0.0082,7806 "p_holm": 0.0164,7807 "significant_holm_0_05": true7808 },7809 "exact_match": {7810 "mean": 0.0,7811 "ci_low": 0.0,7812 "ci_high": 0.0,7813 "n_items": 812,7814 "n_groups": 54,7815 "p_value_two_sided": 0.0,7816 "p_holm": 0.0,7817 "significant_holm_0_05": true7818 },7819 "number_recall": {7820 "mean": -0.011089976986399077,7821 "ci_low": -0.018371211973439192,7822 "ci_high": -0.004966091236818713,7823 "n_items": 812,7824 "n_groups": 54,7825 "p_value_two_sided": 0.0,7826 "p_holm": 0.0,7827 "significant_holm_0_05": true7828 },7829 "placeholders_exact": {7830 "mean": -0.0049261083743842365,7831 "ci_low": -0.015151515151515152,7832 "ci_high": 0.0,7833 "n_items": 812,7834 "n_groups": 54,7835 "p_value_two_sided": 0.0,7836 "p_holm": 0.0,7837 "significant_holm_0_05": true7838 }7839 },7840 "htr-m1-tiny": {7841 "chrf": {7842 "mean": 0.7664004338482392,7843 "ci_low": 0.2656491184452206,7844 "ci_high": 1.163443231020247,7845 "n_items": 812,7846 "n_groups": 54,7847 "p_value_two_sided": 0.0038,7848 "p_holm": 0.0072,7849 "significant_holm_0_05": true7850 },7851 "edit_sim": {7852 "mean": 0.009229355231554435,7853 "ci_low": 0.004083401495819769,7854 "ci_high": 0.014358855630769786,7855 "n_items": 812,7856 "n_groups": 54,7857 "p_value_two_sided": 0.0006,7858 "p_holm": 0.0018,7859 "significant_holm_0_05": true7860 },7861 "rougeL": {7862 "mean": 0.010263430635877086,7863 "ci_low": 0.0037001025157420513,7864 "ci_high": 0.015556704594295928,7865 "n_items": 812,7866 "n_groups": 54,7867 "p_value_two_sided": 0.0036,7868 "p_holm": 0.0072,7869 "significant_holm_0_05": true7870 },7871 "exact_match": {7872 "mean": 0.0,7873 "ci_low": 0.0,7874 "ci_high": 0.0,7875 "n_items": 812,7876 "n_groups": 54,7877 "p_value_two_sided": 0.0,7878 "p_holm": 0.0,7879 "significant_holm_0_05": true7880 },7881 "number_recall": {7882 "mean": -0.008787439298523042,7883 "ci_low": -0.013865984136493224,7884 "ci_high": -0.00473654602156527,7885 "n_items": 812,7886 "n_groups": 54,7887 "p_value_two_sided": 0.0,7888 "p_holm": 0.0,7889 "significant_holm_0_05": true7890 },7891 "placeholders_exact": {7892 "mean": -0.011083743842364532,7893 "ci_low": -0.024581234545288027,7894 "ci_high": -0.0027172991268951696,7895 "n_items": 812,7896 "n_groups": 54,7897 "p_value_two_sided": 0.0,7898 "p_holm": 0.0,7899 "significant_holm_0_05": true7900 }7901 },7902 "htr-m3tokenonly-tiny": {7903 "chrf": {7904 "mean": 0.621995369873511,7905 "ci_low": 0.15571289205753627,7906 "ci_high": 1.063856253249741,7907 "n_items": 812,7908 "n_groups": 54,7909 "p_value_two_sided": 0.0122,7910 "p_holm": 0.0122,7911 "significant_holm_0_05": true7912 },7913 "edit_sim": {7914 "mean": 0.007776402109849017,7915 "ci_low": 0.0028943588206025994,7916 "ci_high": 0.012827568016482693,7917 "n_items": 812,7918 "n_groups": 54,7919 "p_value_two_sided": 0.0018,7920 "p_holm": 0.0054,7921 "significant_holm_0_05": true7922 },7923 "rougeL": {7924 "mean": 0.009705100914996169,7925 "ci_low": 0.00333173627942477,7926 "ci_high": 0.01512653241283484,7927 "n_items": 812,7928 "n_groups": 54,7929 "p_value_two_sided": 0.0044,7930 "p_holm": 0.0088,7931 "significant_holm_0_05": true7932 },7933 "exact_match": {7934 "mean": 0.0,7935 "ci_low": 0.0,7936 "ci_high": 0.0,7937 "n_items": 812,7938 "n_groups": 54,7939 "p_value_two_sided": 0.0,7940 "p_holm": 0.0,7941 "significant_holm_0_05": true7942 },7943 "number_recall": {7944 "mean": -0.0067686312549548235,7945 "ci_low": -0.011507642887981883,7946 "ci_high": -0.003257318776080521,7947 "n_items": 812,7948 "n_groups": 54,7949 "p_value_two_sided": 0.0,7950 "p_holm": 0.0,7951 "significant_holm_0_05": true7952 },7953 "placeholders_exact": {7954 "mean": -0.009852216748768473,7955 "ci_low": -0.03034493223797813,7956 "ci_high": 0.0,7957 "n_items": 812,7958 "n_groups": 54,7959 "p_value_two_sided": 0.0,7960 "p_holm": 0.0,7961 "significant_holm_0_05": true7962 }7963 }7964 },7965 "stylometry": {7966 "rewrite_vs_original": {7967 "W1_mean_sent_len": 0.11074263090089767,7968 "W1_sd_sent_len": 0.06910661758593184,7969 "W1_ttr": 0.46007853018735506,7970 "W1_mtld": 0.6583988872767176,7971 "W1_mean_word_len": 0.3348991362821439,7972 "W1_long_word_rate": 0.1101127464253813,7973 "W1_comma_rate": 0.47070885407597335,7974 "W1_semicolon_rate": 0.0581734178097243,7975 "W1_paren_rate": 0.03939234290734981,7976 "W1_discourse_rate": 0.07468723717617147,7977 "W1_passive_rate": 0.06331375570481547,7978 "W1_nominal_rate": 0.1027393852388131,7979 "W1_first_person_rate": 0.0735359938861468,7980 "W1_subord_rate": 0.1808926799123699,7981 "W1_hedge_rate": 0.07728727676099997,7982 "W1_negation_rate": 0.052357757472774574,7983 "W1_digit_rate": 0.026806789963701214,7984 "W1_flesch": 0.27112769148679694,7985 "W1_mean": 0.17968676283633697986 },7987 "feature_means": {7988 "original": {7989 "mean_sent_len": 25.922313517079502,7990 "sd_sent_len": 8.391208895713133,7991 "ttr": 0.6359301840559946,7992 "mtld": 73.15674679740306,7993 "mean_word_len": 5.588922026829178,7994 "long_word_rate": 0.12776390713147695,7995 "comma_rate": 0.04944856292698259,7996 "semicolon_rate": 0.0019109605953622733,7997 "paren_rate": 0.0156993755935184,7998 "discourse_rate": 0.011167688512642635,7999 "passive_rate": 0.43872504069548346,8000 "nominal_rate": 0.06640595987882356,8001 "first_person_rate": 0.0028538270474617725,8002 "subord_rate": 0.026248179285876524,8003 "hedge_rate": 0.00743891116514551,8004 "negation_rate": 0.005335766696749265,8005 "digit_rate": 0.008834104532639858,8006 "flesch": 16.2065545490268338007 },8008 "rewrite": {8009 "mean_sent_len": 26.526392241447656,8010 "sd_sent_len": 8.411969725005081,8011 "ttr": 0.6800683485939298,8012 "mtld": 92.054882510237,8013 "mean_word_len": 5.730875248373908,8014 "long_word_rate": 0.13178202980314,8015 "comma_rate": 0.06378831170373167,8016 "semicolon_rate": 0.0019767553630901497,8017 "paren_rate": 0.01556086479732947,8018 "discourse_rate": 0.011730896013006906,8019 "passive_rate": 0.4482133322342678,8020 "nominal_rate": 0.06382138002363137,8021 "first_person_rate": 0.0023862233024847736,8022 "subord_rate": 0.0292627273418811,8023 "hedge_rate": 0.007605061210442349,8024 "negation_rate": 0.005637671442888036,8025 "digit_rate": 0.009003819071298401,8026 "flesch": 12.6870984484838168027 }8028 },8029 "feature_sds": {8030 "original": {8031 "mean_sent_len": 6.935150386353656,8032 "sd_sent_len": 4.298391200303736,8033 "ttr": 0.09425629272872084,8034 "mtld": 22.75370152852229,8035 "mean_word_len": 0.4124564876751496,8036 "long_word_rate": 0.041077112789870605,8037 "comma_rate": 0.030190865450344394,8038 "semicolon_rate": 0.005694522793569089,8039 "paren_rate": 0.018134285778734993,8040 "discourse_rate": 0.010291698409833412,8041 "passive_rate": 0.3493573097045577,8042 "nominal_rate": 0.03013506437064267,8043 "first_person_rate": 0.0068185890904091965,8044 "subord_rate": 0.01621128813921175,8045 "hedge_rate": 0.009085448031882506,8046 "negation_rate": 0.007662056450583307,8047 "digit_rate": 0.013538156982454171,8048 "flesch": 12.476359782870688049 },8050 "rewrite": {8051 "mean_sent_len": 6.618487785954692,8052 "sd_sent_len": 4.3600969798838545,8053 "ttr": 0.09277344125532722,8054 "mtld": 31.115086371539014,8055 "mean_word_len": 0.4251793316677886,8056 "long_word_rate": 0.04308078341452632,8057 "comma_rate": 0.029753776840313106,8058 "semicolon_rate": 0.00550884707944613,8059 "paren_rate": 0.018308096906693452,8060 "discourse_rate": 0.010533017245338349,8061 "passive_rate": 0.39742815177660445,8062 "nominal_rate": 0.028914584198177898,8063 "first_person_rate": 0.006154360340379272,8064 "subord_rate": 0.017291119359576203,8065 "hedge_rate": 0.009889862313561912,8066 "negation_rate": 0.008379217497947811,8067 "digit_rate": 0.013800879094384128,8068 "flesch": 13.4068606691435878069 }8070 },8071 "B0_identity": {8072 "vs_original": {8073 "W1_mean_sent_len": 0.11074263090089767,8074 "W1_sd_sent_len": 0.06910661758593184,8075 "W1_ttr": 0.46007853018735506,8076 "W1_mtld": 0.6583988872767176,8077 "W1_mean_word_len": 0.3348991362821439,8078 "W1_long_word_rate": 0.1101127464253813,8079 "W1_comma_rate": 0.47070885407597335,8080 "W1_semicolon_rate": 0.0581734178097243,8081 "W1_paren_rate": 0.03939234290734981,8082 "W1_discourse_rate": 0.07468723717617147,8083 "W1_passive_rate": 0.06331375570481547,8084 "W1_nominal_rate": 0.1027393852388131,8085 "W1_first_person_rate": 0.0735359938861468,8086 "W1_subord_rate": 0.1808926799123699,8087 "W1_hedge_rate": 0.07728727676099997,8088 "W1_negation_rate": 0.052357757472774574,8089 "W1_digit_rate": 0.026806789963701214,8090 "W1_flesch": 0.27112769148679694,8091 "W1_mean": 0.17968676283633698092 },8093 "feature_means": {8094 "mean_sent_len": 26.526392241447656,8095 "sd_sent_len": 8.411969725005081,8096 "ttr": 0.6800683485939298,8097 "mtld": 92.054882510237,8098 "mean_word_len": 5.730875248373908,8099 "long_word_rate": 0.13178202980314,8100 "comma_rate": 0.06378831170373167,8101 "semicolon_rate": 0.0019767553630901497,8102 "paren_rate": 0.01556086479732947,8103 "discourse_rate": 0.011730896013006906,8104 "passive_rate": 0.4482133322342678,8105 "nominal_rate": 0.06382138002363137,8106 "first_person_rate": 0.0023862233024847736,8107 "subord_rate": 0.0292627273418811,8108 "hedge_rate": 0.007605061210442349,8109 "negation_rate": 0.005637671442888036,8110 "digit_rate": 0.009003819071298401,8111 "flesch": 12.6870984484838168112 },8113 "probe": {8114 "probe_auc_orig_vs_rewrite": 0.7194972358051452,8115 "probe_acc_heldout": 0.6598360655737705,8116 "eval_mean_p_original": 0.3879233107186646,8117 "eval_frac_classified_original": 0.30911330049261088118 }8119 },8120 "B1_generic_paraphrase": {8121 "vs_original": {8122 "W1_mean_sent_len": 0.8288758055081382,8123 "W1_sd_sent_len": 0.725366491589682,8124 "W1_ttr": 0.42855125313643194,8125 "W1_mtld": 0.6142087490196315,8126 "W1_mean_word_len": 0.5606382997855508,8127 "W1_long_word_rate": 0.36077155770567926,8128 "W1_comma_rate": 0.46331258168304607,8129 "W1_semicolon_rate": 0.05762275797290836,8130 "W1_paren_rate": 0.040300532212367166,8131 "W1_discourse_rate": 0.07298653716130941,8132 "W1_passive_rate": 0.3240549054951341,8133 "W1_nominal_rate": 0.10505753343434594,8134 "W1_first_person_rate": 0.04903270271383869,8135 "W1_subord_rate": 0.13758047470100743,8136 "W1_hedge_rate": 0.1951491250361723,8137 "W1_negation_rate": 0.050128661064338664,8138 "W1_digit_rate": 0.026995341297396152,8139 "W1_flesch": 0.8636332867329088,8140 "W1_mean": 0.328014810902771468141 },8142 "feature_means": {8143 "mean_sent_len": 31.360843771034645,8144 "sd_sent_len": 11.324073498985973,8145 "ttr": 0.6770566644527949,8146 "mtld": 90.76637731665733,8147 "mean_word_len": 5.826473213324955,8148 "long_word_rate": 0.14279084352558577,8149 "comma_rate": 0.06357290694215817,8150 "semicolon_rate": 0.0019753916671067565,8151 "paren_rate": 0.015508356683635994,8152 "discourse_rate": 0.010566130484187652,8153 "passive_rate": 0.5464093753810502,8154 "nominal_rate": 0.06882257778469811,8155 "first_person_rate": 0.002549659738397044,8156 "subord_rate": 0.028443026186754344,8157 "hedge_rate": 0.0057523410419924835,8158 "negation_rate": 0.005610381404987051,8159 "digit_rate": 0.008861841024039778,8160 "flesch": 5.0678180155686158161 },8162 "probe": {8163 "probe_auc_orig_vs_rewrite": 0.7194972358051452,8164 "probe_acc_heldout": 0.6598360655737705,8165 "eval_mean_p_original": 0.35722670677832524,8166 "eval_frac_classified_original": 0.25615763546798038167 }8168 },8169 "htr-m1-small": {8170 "vs_original": {8171 "W1_mean_sent_len": 0.11152730728453307,8172 "W1_sd_sent_len": 0.08206924464288196,8173 "W1_ttr": 0.2022075401997615,8174 "W1_mtld": 0.20319272284558432,8175 "W1_mean_word_len": 0.07237981222547075,8176 "W1_long_word_rate": 0.06342946896231229,8177 "W1_comma_rate": 0.3362096129097078,8178 "W1_semicolon_rate": 0.06971572794138402,8179 "W1_paren_rate": 0.04376154203260208,8180 "W1_discourse_rate": 0.08996871008145514,8181 "W1_passive_rate": 0.20159618829238843,8182 "W1_nominal_rate": 0.12802350820707634,8183 "W1_first_person_rate": 0.03245297606668454,8184 "W1_subord_rate": 0.12426065570709544,8185 "W1_hedge_rate": 0.08041336449083798,8186 "W1_negation_rate": 0.07388041323976823,8187 "W1_digit_rate": 0.030395030760212157,8188 "W1_flesch": 0.10875900551727707,8189 "W1_mean": 0.114124601744835168190 },8191 "feature_means": {8192 "mean_sent_len": 26.463651007585728,8193 "sd_sent_len": 8.65494078536775,8194 "ttr": 0.655316730558045,8195 "mtld": 78.65683505469799,8196 "mean_word_len": 5.59891081046762,8197 "long_word_rate": 0.1268067225435523,8198 "comma_rate": 0.059621626757749,8199 "semicolon_rate": 0.001592362988427233,8200 "paren_rate": 0.015648451863580777,8201 "discourse_rate": 0.011923303483804762,8202 "passive_rate": 0.5124140449725423,8203 "nominal_rate": 0.06296479529491944,8204 "first_person_rate": 0.0027479961872294836,8205 "subord_rate": 0.028332007686550816,8206 "hedge_rate": 0.0075935752136669105,8207 "negation_rate": 0.005928769646993747,8208 "digit_rate": 0.009046617918344461,8209 "flesch": 15.9764397645173278210 },8211 "probe": {8212 "probe_auc_orig_vs_rewrite": 0.7194972358051452,8213 "probe_acc_heldout": 0.6598360655737705,8214 "eval_mean_p_original": 0.5081863905888679,8215 "eval_frac_classified_original": 0.52463054187192128216 }8217 },8218 "htr-m1-tiny-s3": {8219 "vs_original": {8220 "W1_mean_sent_len": 0.10252097009758403,8221 "W1_sd_sent_len": 0.08430885612227829,8222 "W1_ttr": 0.22992030843303327,8223 "W1_mtld": 0.25321913753331915,8224 "W1_mean_word_len": 0.08288734974739316,8225 "W1_long_word_rate": 0.06828583560128136,8226 "W1_comma_rate": 0.3414146629555512,8227 "W1_semicolon_rate": 0.08959417003713344,8228 "W1_paren_rate": 0.04368102008392133,8229 "W1_discourse_rate": 0.08966372775683097,8230 "W1_passive_rate": 0.20253940120923722,8231 "W1_nominal_rate": 0.11977244316583358,8232 "W1_first_person_rate": 0.02877897133706758,8233 "W1_subord_rate": 0.1272493573402133,8234 "W1_hedge_rate": 0.061684924798706034,8235 "W1_negation_rate": 0.08755215131494483,8236 "W1_digit_rate": 0.033137676187958195,8237 "W1_flesch": 0.09610318407928223,8238 "W1_mean": 0.119017452655642738239 },8240 "feature_means": {8241 "mean_sent_len": 26.38764550822743,8242 "sd_sent_len": 8.542026659873681,8243 "ttr": 0.6580069161187565,8244 "mtld": 80.22056606592284,8245 "mean_word_len": 5.601363505579735,8246 "long_word_rate": 0.12647166659958145,8247 "comma_rate": 0.05978012430851837,8248 "semicolon_rate": 0.0014816408906348528,8249 "paren_rate": 0.015725699375810823,8250 "discourse_rate": 0.011904333965439601,8251 "passive_rate": 0.5136343985789793,8252 "nominal_rate": 0.06318107427289132,8253 "first_person_rate": 0.0028170878222525133,8254 "subord_rate": 0.028344048151260067,8255 "hedge_rate": 0.007823693830867838,8256 "negation_rate": 0.006004759820441787,8257 "digit_rate": 0.009118251625633587,8258 "flesch": 15.8044548064434848259 },8260 "probe": {8261 "probe_auc_orig_vs_rewrite": 0.7194972358051452,8262 "probe_acc_heldout": 0.6598360655737705,8263 "eval_mean_p_original": 0.49785055220924035,8264 "eval_frac_classified_original": 0.488916256157635458265 }8266 },8267 "htr-m1-tiny": {8268 "vs_original": {8269 "W1_mean_sent_len": 0.10299240024245253,8270 "W1_sd_sent_len": 0.09417150396934933,8271 "W1_ttr": 0.21654625257353977,8272 "W1_mtld": 0.252210060548455,8273 "W1_mean_word_len": 0.07800909633535662,8274 "W1_long_word_rate": 0.07238942284274409,8275 "W1_comma_rate": 0.35074099494636746,8276 "W1_semicolon_rate": 0.10814262826877964,8277 "W1_paren_rate": 0.038576450872248404,8278 "W1_discourse_rate": 0.1048269751219622,8279 "W1_passive_rate": 0.17190506120121102,8280 "W1_nominal_rate": 0.11370895726937875,8281 "W1_first_person_rate": 0.030155397732341608,8282 "W1_subord_rate": 0.09890156025840674,8283 "W1_hedge_rate": 0.06269079667719703,8284 "W1_negation_rate": 0.0560706049583465,8285 "W1_digit_rate": 0.02935087000433156,8286 "W1_flesch": 0.09956500398993463,8287 "W1_mean": 0.11560855765624468288 },8289 "feature_means": {8290 "mean_sent_len": 26.410427428346154,8291 "sd_sent_len": 8.585581683165033,8292 "ttr": 0.6566981313762378,8293 "mtld": 80.08504749451828,8294 "mean_word_len": 5.604071898122027,8295 "long_word_rate": 0.12654406921546288,8296 "comma_rate": 0.06023833480432478,8297 "semicolon_rate": 0.0013870811728544177,8298 "paren_rate": 0.015731237186747637,8299 "discourse_rate": 0.011979975150884089,8300 "passive_rate": 0.5030011928410942,8301 "nominal_rate": 0.06343405740309377,8302 "first_person_rate": 0.002836533945379594,8303 "subord_rate": 0.027848526165721345,8304 "hedge_rate": 0.00737018382392236,8305 "negation_rate": 0.005764926192940201,8306 "digit_rate": 0.009082942647083663,8307 "flesch": 15.6680973233890828308 },8309 "probe": {8310 "probe_auc_orig_vs_rewrite": 0.7194972358051452,8311 "probe_acc_heldout": 0.6598360655737705,8312 "eval_mean_p_original": 0.4953991398670471,8313 "eval_frac_classified_original": 0.490147783251231548314 }8315 },8316 "htr-m3tokenonly-tiny": {8317 "vs_original": {8318 "W1_mean_sent_len": 0.1019753846900324,8319 "W1_sd_sent_len": 0.08240075678930127,8320 "W1_ttr": 0.27501212251762136,8321 "W1_mtld": 0.31112037198181647,8322 "W1_mean_word_len": 0.11531916626615175,8323 "W1_long_word_rate": 0.0740686890527514,8324 "W1_comma_rate": 0.33041450851842874,8325 "W1_semicolon_rate": 0.10106528421948523,8326 "W1_paren_rate": 0.0408216017896686,8327 "W1_discourse_rate": 0.1205475015071924,8328 "W1_passive_rate": 0.18051041857489933,8329 "W1_nominal_rate": 0.11975907344170776,8330 "W1_first_person_rate": 0.037944164852634674,8331 "W1_subord_rate": 0.13703863675213288,8332 "W1_hedge_rate": 0.0660361580365902,8333 "W1_negation_rate": 0.09083458700730451,8334 "W1_digit_rate": 0.033687813459038846,8335 "W1_flesch": 0.12271549923136899,8336 "W1_mean": 0.130070652149340358337 },8338 "feature_means": {8339 "mean_sent_len": 26.366780564810142,8340 "sd_sent_len": 8.55971467906528,8341 "ttr": 0.662261078888523,8342 "mtld": 82.00598746296612,8343 "mean_word_len": 5.627481813801742,8344 "long_word_rate": 0.12738804478170226,8345 "comma_rate": 0.05937310613504791,8346 "semicolon_rate": 0.001415349748591214,8347 "paren_rate": 0.0158243768141978,8348 "discourse_rate": 0.012083769223814315,8349 "passive_rate": 0.506104937755184,8350 "nominal_rate": 0.06325323210229362,8351 "first_person_rate": 0.0027308569881101317,8352 "subord_rate": 0.028510645476626847,8353 "hedge_rate": 0.007268003000351876,8354 "negation_rate": 0.0060395919167252924,8355 "digit_rate": 0.00916976312090791,8356 "flesch": 15.0743414514892458357 },8358 "probe": {8359 "probe_auc_orig_vs_rewrite": 0.7194972358051452,8360 "probe_acc_heldout": 0.6598360655737705,8361 "eval_mean_p_original": 0.48143355617514255,8362 "eval_frac_classified_original": 0.46059113300492618363 }8364 }8365 },8366 "confirmatory": {8367 "preregistration_frozen": true,8368 "n_boot": 100008369 }8370}