Repository Public Research preview
1{2 "created_at": "2026-11-09T17:33:43Z",3 "git": {4 "commit": "43899c8924172a93eb7fdb44567c22c0c6bb1e47",5 "branch": "main",6 "dirty": true7 },8 "libraries": {9 "python": "3.14.4",10 "torch": "2.12.0",11 "numpy": "2.5.4",12 "anthropic": "2.3.0",13 "transformers": "5.8.1",14 "sentencepiece": "0.2.2",15 "lxml": "6.1.3",16 "pyarrow": "24.0.0",17 "sacrebleu": "2.6.0",18 "datasketch": "2.0.0",19 "rapidfuzz": "3.14.6"20 },21 "hardware": {22 "platform": "macOS-27.0-arm64-arm-64bit-Mach-O",23 "machine": "arm64",24 "cpu_count": 18,25 "chip": "Apple M5 Max",26 "ram_gb": 48.0,27 "torch_mps": true,28 "torch_cuda": false29 },30 "n_pairs": 26,31 "systems": {32 "B0_identity": {33 "overall": {34 "n": 26,35 "chrf": {36 "mean": 91.1404460841871,37 "ci_low": 90.12171941530144,38 "ci_high": 94.03846235532119,39 "n_items": 26,40 "n_groups": 441 },42 "edit_sim": {43 "mean": 0.8449045790299752,44 "ci_low": 0.8093012950979083,45 "ci_high": 0.8803982625263892,46 "n_items": 26,47 "n_groups": 448 },49 "rougeL": {50 "mean": 0.8792542397834846,51 "ci_low": 0.8599093840588314,52 "ci_high": 0.8995957102958096,53 "n_items": 26,54 "n_groups": 455 },56 "exact_match": {57 "mean": 0.0,58 "ci_low": 0.0,59 "ci_high": 0.0,60 "n_items": 26,61 "n_groups": 462 },63 "number_recall": {64 "mean": 1.0,65 "ci_low": 1.0,66 "ci_high": 1.0,67 "n_items": 26,68 "n_groups": 469 },70 "placeholders_exact": {71 "mean": 1.0,72 "ci_low": 1.0,73 "ci_high": 1.0,74 "n_items": 26,75 "n_groups": 476 }77 },78 "by_variant": {79 "A": {80 "n": 5,81 "chrf": {82 "mean": 91.56726567266112,83 "ci_low": 89.67139522833332,84 "ci_high": 93.99834671555749,85 "n_items": 5,86 "n_groups": 487 },88 "edit_sim": {89 "mean": 0.8147362968945098,90 "ci_low": 0.647677334078258,91 "ci_high": 0.879916537975881,92 "n_items": 5,93 "n_groups": 494 },95 "rougeL": {96 "mean": 0.8695613528443976,97 "ci_low": 0.783960720130933,98 "ci_high": 0.9139159010126752,99 "n_items": 5,100 "n_groups": 4101 },102 "exact_match": {103 "mean": 0.0,104 "ci_low": 0.0,105 "ci_high": 0.0,106 "n_items": 5,107 "n_groups": 4108 },109 "number_recall": {110 "mean": 1.0,111 "ci_low": 1.0,112 "ci_high": 1.0,113 "n_items": 5,114 "n_groups": 4115 },116 "placeholders_exact": {117 "mean": 1.0,118 "ci_low": 1.0,119 "ci_high": 1.0,120 "n_items": 5,121 "n_groups": 4122 }123 },124 "B": {125 "n": 4,126 "chrf": {127 "mean": 89.98839323272541,128 "ci_low": 89.0959820465978,129 "ci_high": 91.43155358009183,130 "n_items": 4,131 "n_groups": 3132 },133 "edit_sim": {134 "mean": 0.917194624608786,135 "ci_low": 0.8998110376003149,136 "ci_high": 0.9436008676789588,137 "n_items": 4,138 "n_groups": 3139 },140 "rougeL": {141 "mean": 0.9109665600319061,142 "ci_low": 0.9013980868285504,143 "ci_high": 0.927536231884058,144 "n_items": 4,145 "n_groups": 3146 },147 "exact_match": {148 "mean": 0.0,149 "ci_low": 0.0,150 "ci_high": 0.0,151 "n_items": 4,152 "n_groups": 3153 },154 "number_recall": {155 "mean": 1.0,156 "ci_low": 1.0,157 "ci_high": 1.0,158 "n_items": 4,159 "n_groups": 3160 },161 "placeholders_exact": {162 "mean": 1.0,163 "ci_low": 1.0,164 "ci_high": 1.0,165 "n_items": 4,166 "n_groups": 3167 }168 },169 "C": {170 "n": 4,171 "chrf": {172 "mean": 89.75930251003241,173 "ci_low": 89.51306930844578,174 "ci_high": 90.4945773245641,175 "n_items": 4,176 "n_groups": 3177 },178 "edit_sim": {179 "mean": 0.9352156491009707,180 "ci_low": 0.9306326304106548,181 "ci_high": 0.9370932754880693,182 "n_items": 4,183 "n_groups": 3184 },185 "rougeL": {186 "mean": 0.921832073708196,187 "ci_low": 0.9172932330827067,188 "ci_high": 0.9245858761987794,189 "n_items": 4,190 "n_groups": 3191 },192 "exact_match": {193 "mean": 0.0,194 "ci_low": 0.0,195 "ci_high": 0.0,196 "n_items": 4,197 "n_groups": 3198 },199 "number_recall": {200 "mean": 1.0,201 "ci_low": 1.0,202 "ci_high": 1.0,203 "n_items": 4,204 "n_groups": 3205 },206 "placeholders_exact": {207 "mean": 1.0,208 "ci_low": 1.0,209 "ci_high": 1.0,210 "n_items": 4,211 "n_groups": 3212 }213 },214 "D": {215 "n": 4,216 "chrf": {217 "mean": 90.28491940536668,218 "ci_low": 89.62912454069641,219 "ci_high": 91.14660547185507,220 "n_items": 4,221 "n_groups": 3222 },223 "edit_sim": {224 "mean": 0.9330933245913863,225 "ci_low": 0.9298440979955457,226 "ci_high": 0.9392624728850324,227 "n_items": 4,228 "n_groups": 3229 },230 "rougeL": {231 "mean": 0.9184536953298176,232 "ci_low": 0.9172932330827067,233 "ci_high": 0.920863309352518,234 "n_items": 4,235 "n_groups": 3236 },237 "exact_match": {238 "mean": 0.0,239 "ci_low": 0.0,240 "ci_high": 0.0,241 "n_items": 4,242 "n_groups": 3243 },244 "number_recall": {245 "mean": 1.0,246 "ci_low": 1.0,247 "ci_high": 1.0,248 "n_items": 4,249 "n_groups": 3250 },251 "placeholders_exact": {252 "mean": 1.0,253 "ci_low": 1.0,254 "ci_high": 1.0,255 "n_items": 4,256 "n_groups": 3257 }258 },259 "E": {260 "n": 9,261 "chrf": {262 "mean": 92.40942324811789,263 "ci_low": 90.60277419456195,264 "ci_high": 95.04836125176206,265 "n_items": 9,266 "n_groups": 4267 },268 "edit_sim": {269 "mean": 0.7502025752335819,270 "ci_low": 0.6716269534774902,271 "ci_high": 0.8484221024286961,272 "n_items": 9,273 "n_groups": 4274 },275 "rougeL": {276 "mean": 0.8341993504298811,277 "ci_low": 0.7938844875731271,278 "ci_high": 0.8845929290008238,279 "n_items": 9,280 "n_groups": 4281 },282 "exact_match": {283 "mean": 0.0,284 "ci_low": 0.0,285 "ci_high": 0.0,286 "n_items": 9,287 "n_groups": 4288 },289 "number_recall": {290 "mean": 1.0,291 "ci_low": 1.0,292 "ci_high": 1.0,293 "n_items": 9,294 "n_groups": 4295 },296 "placeholders_exact": {297 "mean": 1.0,298 "ci_low": 1.0,299 "ci_high": 1.0,300 "n_items": 9,301 "n_groups": 4302 }303 }304 },305 "by_discipline": {306 "biomedicine": {307 "n": 26,308 "chrf": {309 "mean": 91.1404460841871,310 "ci_low": 90.12171941530144,311 "ci_high": 94.03846235532119,312 "n_items": 26,313 "n_groups": 4314 },315 "edit_sim": {316 "mean": 0.8449045790299752,317 "ci_low": 0.8093012950979083,318 "ci_high": 0.8803982625263892,319 "n_items": 26,320 "n_groups": 4321 },322 "rougeL": {323 "mean": 0.8792542397834846,324 "ci_low": 0.8599093840588314,325 "ci_high": 0.8995957102958096,326 "n_items": 26,327 "n_groups": 4328 },329 "exact_match": {330 "mean": 0.0,331 "ci_low": 0.0,332 "ci_high": 0.0,333 "n_items": 26,334 "n_groups": 4335 },336 "number_recall": {337 "mean": 1.0,338 "ci_low": 1.0,339 "ci_high": 1.0,340 "n_items": 26,341 "n_groups": 4342 },343 "placeholders_exact": {344 "mean": 1.0,345 "ci_low": 1.0,346 "ci_high": 1.0,347 "n_items": 26,348 "n_groups": 4349 }350 }351 },352 "by_era": {353 "2000-2009": {354 "n": 17,355 "chrf": {356 "mean": 90.25355223859705,357 "ci_low": 89.93731918711131,358 "ci_high": 90.83331283298759,359 "n_items": 17,360 "n_groups": 2361 },362 "edit_sim": {363 "mean": 0.8637533764057753,364 "ci_low": 0.8476423352001958,365 "ci_high": 0.8932902852826711,366 "n_items": 17,367 "n_groups": 2368 },369 "rougeL": {370 "mean": 0.886338522357529,371 "ci_low": 0.8797897448826736,372 "ci_high": 0.8983446143947633,373 "n_items": 17,374 "n_groups": 2375 },376 "exact_match": {377 "mean": 0.0,378 "ci_low": 0.0,379 "ci_high": 0.0,380 "n_items": 17,381 "n_groups": 2382 },383 "number_recall": {384 "mean": 1.0,385 "ci_low": 1.0,386 "ci_high": 1.0,387 "n_items": 17,388 "n_groups": 2389 },390 "placeholders_exact": {391 "mean": 1.0,392 "ci_low": 1.0,393 "ci_high": 1.0,394 "n_items": 17,395 "n_groups": 2396 }397 },398 "2010-2017": {399 "n": 9,400 "chrf": {401 "mean": 92.81569001474605,402 "ci_low": 91.13592067034733,403 "ci_high": 96.17522870354355,404 "n_items": 9,405 "n_groups": 2406 },407 "edit_sim": {408 "mean": 0.8093012950979084,409 "ci_low": 0.7866448341399499,410 "ci_high": 0.8546142170138253,411 "n_items": 9,412 "n_groups": 2413 },414 "rougeL": {415 "mean": 0.8658728171436229,416 "ci_low": 0.8477602746664833,417 "ci_high": 0.9020979020979021,418 "n_items": 9,419 "n_groups": 2420 },421 "exact_match": {422 "mean": 0.0,423 "ci_low": 0.0,424 "ci_high": 0.0,425 "n_items": 9,426 "n_groups": 2427 },428 "number_recall": {429 "mean": 1.0,430 "ci_low": 1.0,431 "ci_high": 1.0,432 "n_items": 9,433 "n_groups": 2434 },435 "placeholders_exact": {436 "mean": 1.0,437 "ci_low": 1.0,438 "ci_high": 1.0,439 "n_items": 9,440 "n_groups": 2441 }442 }443 },444 "by_role": {445 "abstract": {446 "n": 6,447 "chrf": {448 "mean": 90.8333128329876,449 "ci_low": 90.8333128329876,450 "ci_high": 90.8333128329876,451 "n_items": 6,452 "n_groups": 1453 },454 "edit_sim": {455 "mean": 0.8932902852826708,456 "ci_low": 0.8932902852826708,457 "ci_high": 0.8932902852826708,458 "n_items": 6,459 "n_groups": 1460 },461 "rougeL": {462 "mean": 0.8983446143947633,463 "ci_low": 0.8983446143947633,464 "ci_high": 0.8983446143947633,465 "n_items": 6,466 "n_groups": 1467 },468 "exact_match": {469 "mean": 0.0,470 "ci_low": 0.0,471 "ci_high": 0.0,472 "n_items": 6,473 "n_groups": 1474 },475 "number_recall": {476 "mean": 1.0,477 "ci_low": 1.0,478 "ci_high": 1.0,479 "n_items": 6,480 "n_groups": 1481 },482 "placeholders_exact": {483 "mean": 1.0,484 "ci_low": 1.0,485 "ci_high": 1.0,486 "n_items": 6,487 "n_groups": 1488 }489 },490 "data": {491 "n": 9,492 "chrf": {493 "mean": 92.28671167864348,494 "ci_low": 90.34245316619344,495 "ci_high": 96.17522870354355,496 "n_items": 9,497 "n_groups": 2498 },499 "edit_sim": {500 "mean": 0.803851490501006,501 "ci_low": 0.7784701272445963,502 "ci_high": 0.8546142170138253,503 "n_items": 9,504 "n_groups": 2505 },506 "rougeL": {507 "mean": 0.8722145245324053,508 "ci_low": 0.8572728357496571,509 "ci_high": 0.9020979020979021,510 "n_items": 9,511 "n_groups": 2512 },513 "exact_match": {514 "mean": 0.0,515 "ci_low": 0.0,516 "ci_high": 0.0,517 "n_items": 9,518 "n_groups": 2519 },520 "number_recall": {521 "mean": 1.0,522 "ci_low": 1.0,523 "ci_high": 1.0,524 "n_items": 9,525 "n_groups": 2526 },527 "placeholders_exact": {528 "mean": 1.0,529 "ci_low": 1.0,530 "ci_high": 1.0,531 "n_items": 9,532 "n_groups": 2533 }534 },535 "discussion": {536 "n": 5,537 "chrf": {538 "mean": 89.45115841221275,539 "ci_low": 89.45115841221275,540 "ci_high": 89.45115841221275,541 "n_items": 5,542 "n_groups": 1543 },544 "edit_sim": {545 "mean": 0.9306489847469154,546 "ci_low": 0.9306489847469154,547 "ci_high": 0.9306489847469154,548 "n_items": 5,549 "n_groups": 1550 },551 "rougeL": {552 "mean": 0.9068100358422939,553 "ci_low": 0.9068100358422939,554 "ci_high": 0.9068100358422939,555 "n_items": 5,556 "n_groups": 1557 },558 "exact_match": {559 "mean": 0.0,560 "ci_low": 0.0,561 "ci_high": 0.0,562 "n_items": 5,563 "n_groups": 1564 },565 "number_recall": {566 "mean": 1.0,567 "ci_low": 1.0,568 "ci_high": 1.0,569 "n_items": 5,570 "n_groups": 1571 },572 "placeholders_exact": {573 "mean": 1.0,574 "ci_low": 1.0,575 "ci_high": 1.0,576 "n_items": 5,577 "n_groups": 1578 }579 },580 "related_work": {581 "n": 6,582 "chrf": {583 "mean": 91.13592067034733,584 "ci_low": 91.13592067034733,585 "ci_high": 91.13592067034733,586 "n_items": 6,587 "n_groups": 1588 },589 "edit_sim": {590 "mean": 0.7866448341399499,591 "ci_low": 0.7866448341399499,592 "ci_high": 0.7866448341399499,593 "n_items": 6,594 "n_groups": 1595 },596 "rougeL": {597 "mean": 0.8477602746664834,598 "ci_low": 0.8477602746664834,599 "ci_high": 0.8477602746664834,600 "n_items": 6,601 "n_groups": 1602 },603 "exact_match": {604 "mean": 0.0,605 "ci_low": 0.0,606 "ci_high": 0.0,607 "n_items": 6,608 "n_groups": 1609 },610 "number_recall": {611 "mean": 1.0,612 "ci_low": 1.0,613 "ci_high": 1.0,614 "n_items": 6,615 "n_groups": 1616 },617 "placeholders_exact": {618 "mean": 1.0,619 "ci_low": 1.0,620 "ci_high": 1.0,621 "n_items": 6,622 "n_groups": 1623 }624 }625 }626 },627 "B1_generic_paraphrase": {628 "overall": {629 "n": 26,630 "chrf": {631 "mean": 89.59571236683487,632 "ci_low": 88.33468415694813,633 "ci_high": 93.18832206413845,634 "n_items": 26,635 "n_groups": 4636 },637 "edit_sim": {638 "mean": 0.7658282912919349,639 "ci_low": 0.7023166602092793,640 "ci_high": 0.8149826441758461,641 "n_items": 26,642 "n_groups": 4643 },644 "rougeL": {645 "mean": 0.825234647780773,646 "ci_low": 0.7902203103954746,647 "ci_high": 0.8607410180418086,648 "n_items": 26,649 "n_groups": 4650 },651 "exact_match": {652 "mean": 0.0,653 "ci_low": 0.0,654 "ci_high": 0.0,655 "n_items": 26,656 "n_groups": 4657 },658 "number_recall": {659 "mean": 1.0,660 "ci_low": 1.0,661 "ci_high": 1.0,662 "n_items": 26,663 "n_groups": 4664 },665 "placeholders_exact": {666 "mean": 1.0,667 "ci_low": 1.0,668 "ci_high": 1.0,669 "n_items": 26,670 "n_groups": 4671 }672 },673 "by_variant": {674 "A": {675 "n": 5,676 "chrf": {677 "mean": 89.92213648130267,678 "ci_low": 87.62739537403381,679 "ci_high": 93.5021418022875,680 "n_items": 5,681 "n_groups": 4682 },683 "edit_sim": {684 "mean": 0.8076109329429046,685 "ci_low": 0.6511595283877457,686 "ci_high": 0.8727270967889497,687 "n_items": 5,688 "n_groups": 4689 },690 "rougeL": {691 "mean": 0.8602654435684032,692 "ci_low": 0.7919203099059215,693 "ci_high": 0.9047615854067468,694 "n_items": 5,695 "n_groups": 4696 },697 "exact_match": {698 "mean": 0.0,699 "ci_low": 0.0,700 "ci_high": 0.0,701 "n_items": 5,702 "n_groups": 4703 },704 "number_recall": {705 "mean": 1.0,706 "ci_low": 1.0,707 "ci_high": 1.0,708 "n_items": 5,709 "n_groups": 4710 },711 "placeholders_exact": {712 "mean": 1.0,713 "ci_low": 1.0,714 "ci_high": 1.0,715 "n_items": 5,716 "n_groups": 4717 }718 },719 "B": {720 "n": 4,721 "chrf": {722 "mean": 88.92145824388503,723 "ci_low": 88.05384158044586,724 "ci_high": 90.33005525761423,725 "n_items": 4,726 "n_groups": 3727 },728 "edit_sim": {729 "mean": 0.8277130584395712,730 "ci_low": 0.7946006167586314,731 "ci_high": 0.9255555555555556,732 "n_items": 4,733 "n_groups": 3734 },735 "rougeL": {736 "mean": 0.8365592272744438,737 "ci_low": 0.7888153053715966,738 "ci_high": 0.9135338345864662,739 "n_items": 4,740 "n_groups": 3741 },742 "exact_match": {743 "mean": 0.0,744 "ci_low": 0.0,745 "ci_high": 0.0,746 "n_items": 4,747 "n_groups": 3748 },749 "number_recall": {750 "mean": 1.0,751 "ci_low": 1.0,752 "ci_high": 1.0,753 "n_items": 4,754 "n_groups": 3755 },756 "placeholders_exact": {757 "mean": 1.0,758 "ci_low": 1.0,759 "ci_high": 1.0,760 "n_items": 4,761 "n_groups": 3762 }763 },764 "C": {765 "n": 4,766 "chrf": {767 "mean": 88.28196838491945,768 "ci_low": 87.5653982087341,769 "ci_high": 89.51649409867404,770 "n_items": 4,771 "n_groups": 3772 },773 "edit_sim": {774 "mean": 0.8019831542702301,775 "ci_low": 0.725,776 "ci_high": 0.9370932754880693,777 "n_items": 4,778 "n_groups": 3779 },780 "rougeL": {781 "mean": 0.8393793202028108,782 "ci_low": 0.763157894736842,783 "ci_high": 0.920863309352518,784 "n_items": 4,785 "n_groups": 3786 },787 "exact_match": {788 "mean": 0.0,789 "ci_low": 0.0,790 "ci_high": 0.0,791 "n_items": 4,792 "n_groups": 3793 },794 "number_recall": {795 "mean": 1.0,796 "ci_low": 1.0,797 "ci_high": 1.0,798 "n_items": 4,799 "n_groups": 3800 },801 "placeholders_exact": {802 "mean": 1.0,803 "ci_low": 1.0,804 "ci_high": 1.0,805 "n_items": 4,806 "n_groups": 3807 }808 },809 "D": {810 "n": 4,811 "chrf": {812 "mean": 89.3411442757943,813 "ci_low": 88.93380036665171,814 "ci_high": 90.11856583072493,815 "n_items": 4,816 "n_groups": 3817 },818 "edit_sim": {819 "mean": 0.7888262857451942,820 "ci_low": 0.7353579175704988,821 "ci_high": 0.8299736127051393,822 "n_items": 4,823 "n_groups": 3824 },825 "rougeL": {826 "mean": 0.8265881915938101,827 "ci_low": 0.7194244604316546,828 "ci_high": 0.8705318221447254,829 "n_items": 4,830 "n_groups": 3831 },832 "exact_match": {833 "mean": 0.0,834 "ci_low": 0.0,835 "ci_high": 0.0,836 "n_items": 4,837 "n_groups": 3838 },839 "number_recall": {840 "mean": 1.0,841 "ci_low": 1.0,842 "ci_high": 1.0,843 "n_items": 4,844 "n_groups": 3845 },846 "placeholders_exact": {847 "mean": 1.0,848 "ci_low": 1.0,849 "ci_high": 1.0,850 "n_items": 4,851 "n_groups": 3852 }853 },854 "E": {855 "n": 9,856 "chrf": {857 "mean": 90.41106172364425,858 "ci_low": 88.06232057236436,859 "ci_high": 94.51473434132083,860 "n_items": 9,861 "n_groups": 4862 },863 "edit_sim": {864 "mean": 0.6888212127839781,865 "ci_low": 0.5580453259236822,866 "ci_high": 0.7984080552004649,867 "n_items": 9,868 "n_groups": 4869 },870 "rougeL": {871 "mean": 0.7938518520193139,872 "ci_low": 0.7230054161372379,873 "ci_high": 0.8554246780053232,874 "n_items": 9,875 "n_groups": 4876 },877 "exact_match": {878 "mean": 0.0,879 "ci_low": 0.0,880 "ci_high": 0.0,881 "n_items": 9,882 "n_groups": 4883 },884 "number_recall": {885 "mean": 1.0,886 "ci_low": 1.0,887 "ci_high": 1.0,888 "n_items": 9,889 "n_groups": 4890 },891 "placeholders_exact": {892 "mean": 1.0,893 "ci_low": 1.0,894 "ci_high": 1.0,895 "n_items": 9,896 "n_groups": 4897 }898 }899 },900 "by_discipline": {901 "biomedicine": {902 "n": 26,903 "chrf": {904 "mean": 89.59571236683487,905 "ci_low": 88.33468415694813,906 "ci_high": 93.18832206413845,907 "n_items": 26,908 "n_groups": 4909 },910 "edit_sim": {911 "mean": 0.7658282912919349,912 "ci_low": 0.7023166602092793,913 "ci_high": 0.8149826441758461,914 "n_items": 26,915 "n_groups": 4916 },917 "rougeL": {918 "mean": 0.825234647780773,919 "ci_low": 0.7902203103954746,920 "ci_high": 0.8607410180418086,921 "n_items": 26,922 "n_groups": 4923 },924 "exact_match": {925 "mean": 0.0,926 "ci_low": 0.0,927 "ci_high": 0.0,928 "n_items": 26,929 "n_groups": 4930 },931 "number_recall": {932 "mean": 1.0,933 "ci_low": 1.0,934 "ci_high": 1.0,935 "n_items": 26,936 "n_groups": 4937 },938 "placeholders_exact": {939 "mean": 1.0,940 "ci_low": 1.0,941 "ci_high": 1.0,942 "n_items": 26,943 "n_groups": 4944 }945 }946 },947 "by_era": {948 "2000-2009": {949 "n": 17,950 "chrf": {951 "mean": 88.74794487040538,952 "ci_low": 88.13107800344854,953 "ci_high": 89.87886745982628,954 "n_items": 17,955 "n_groups": 2956 },957 "edit_sim": {958 "mean": 0.7834996091993124,959 "ci_low": 0.777135655440652,960 "ci_high": 0.7951668577568564,961 "n_items": 17,962 "n_groups": 2963 },964 "rougeL": {965 "mean": 0.8306095803138808,966 "ci_low": 0.826903567450459,967 "ci_high": 0.8374039372301537,968 "n_items": 17,969 "n_groups": 2970 },971 "exact_match": {972 "mean": 0.0,973 "ci_low": 0.0,974 "ci_high": 0.0,975 "n_items": 17,976 "n_groups": 2977 },978 "number_recall": {979 "mean": 1.0,980 "ci_low": 1.0,981 "ci_high": 1.0,982 "n_items": 17,983 "n_groups": 2984 },985 "placeholders_exact": {986 "mean": 1.0,987 "ci_low": 1.0,988 "ci_high": 1.0,989 "n_items": 17,990 "n_groups": 2991 }992 },993 "2010-2017": {994 "n": 9,995 "chrf": {996 "mean": 91.19705097120169,997 "ci_low": 88.70796210503075,998 "ci_high": 96.17522870354355,999 "n_items": 9,1000 "n_groups": 21001 },1002 "edit_sim": {1003 "mean": 0.7324491352446663,1004 "ci_low": 0.6713665943600868,1005 "ci_high": 0.8546142170138253,1006 "n_items": 9,1007 "n_groups": 21008 },1009 "rougeL": {1010 "mean": 0.8150819974404586,1011 "ci_low": 0.7715740451117367,1012 "ci_high": 0.9020979020979021,1013 "n_items": 9,1014 "n_groups": 21015 },1016 "exact_match": {1017 "mean": 0.0,1018 "ci_low": 0.0,1019 "ci_high": 0.0,1020 "n_items": 9,1021 "n_groups": 21022 },1023 "number_recall": {1024 "mean": 1.0,1025 "ci_low": 1.0,1026 "ci_high": 1.0,1027 "n_items": 9,1028 "n_groups": 21029 },1030 "placeholders_exact": {1031 "mean": 1.0,1032 "ci_low": 1.0,1033 "ci_high": 1.0,1034 "n_items": 9,1035 "n_groups": 21036 }1037 }1038 },1039 "by_role": {1040 "abstract": {1041 "n": 6,1042 "chrf": {1043 "mean": 89.87886745982631,1044 "ci_low": 89.87886745982631,1045 "ci_high": 89.87886745982631,1046 "n_items": 6,1047 "n_groups": 11048 },1049 "edit_sim": {1050 "mean": 0.7951668577568564,1051 "ci_low": 0.7951668577568564,1052 "ci_high": 0.7951668577568564,1053 "n_items": 6,1054 "n_groups": 11055 },1056 "rougeL": {1057 "mean": 0.8374039372301537,1058 "ci_low": 0.8374039372301537,1059 "ci_high": 0.8374039372301537,1060 "n_items": 6,1061 "n_groups": 11062 },1063 "exact_match": {1064 "mean": 0.0,1065 "ci_low": 0.0,1066 "ci_high": 0.0,1067 "n_items": 6,1068 "n_groups": 11069 },1070 "number_recall": {1071 "mean": 1.0,1072 "ci_low": 1.0,1073 "ci_high": 1.0,1074 "n_items": 6,1075 "n_groups": 11076 },1077 "placeholders_exact": {1078 "mean": 1.0,1079 "ci_low": 1.0,1080 "ci_high": 1.0,1081 "n_items": 6,1082 "n_groups": 11083 }1084 },1085 "data": {1086 "n": 9,1087 "chrf": {1088 "mean": 90.4318812104085,1089 "ci_low": 87.56020746384098,1090 "ci_high": 96.17522870354355,1091 "n_items": 9,1092 "n_groups": 21093 },1094 "edit_sim": {1095 "mean": 0.7194759767313297,1096 "ci_low": 0.651906856590082,1097 "ci_high": 0.8546142170138253,1098 "n_items": 9,1099 "n_groups": 21100 },1101 "rougeL": {1102 "mean": 0.8111600926170461,1103 "ci_low": 0.7656911878766183,1104 "ci_high": 0.9020979020979021,1105 "n_items": 9,1106 "n_groups": 21107 },1108 "exact_match": {1109 "mean": 0.0,1110 "ci_low": 0.0,1111 "ci_high": 0.0,1112 "n_items": 9,1113 "n_groups": 21114 },1115 "number_recall": {1116 "mean": 1.0,1117 "ci_low": 1.0,1118 "ci_high": 1.0,1119 "n_items": 9,1120 "n_groups": 21121 },1122 "placeholders_exact": {1123 "mean": 1.0,1124 "ci_low": 1.0,1125 "ci_high": 1.0,1126 "n_items": 9,1127 "n_groups": 21128 }1129 },1130 "discussion": {1131 "n": 5,1132 "chrf": {1133 "mean": 88.81612265097758,1134 "ci_low": 88.81612265097758,1135 "ci_high": 88.81612265097758,1136 "n_items": 5,1137 "n_groups": 11138 },1139 "edit_sim": {1140 "mean": 0.9274102140613362,1141 "ci_low": 0.9274102140613362,1142 "ci_high": 0.9274102140613362,1143 "n_items": 5,1144 "n_groups": 11145 },1146 "rougeL": {1147 "mean": 0.9003584229390681,1148 "ci_low": 0.9003584229390681,1149 "ci_high": 0.9003584229390681,1150 "n_items": 5,1151 "n_groups": 11152 },1153 "exact_match": {1154 "mean": 0.0,1155 "ci_low": 0.0,1156 "ci_high": 0.0,1157 "n_items": 5,1158 "n_groups": 11159 },1160 "number_recall": {1161 "mean": 1.0,1162 "ci_low": 1.0,1163 "ci_high": 1.0,1164 "n_items": 5,1165 "n_groups": 11166 },1167 "placeholders_exact": {1168 "mean": 1.0,1169 "ci_low": 1.0,1170 "ci_high": 1.0,1171 "n_items": 5,1172 "n_groups": 11173 }1174 },1175 "related_work": {1176 "n": 6,1177 "chrf": {1178 "mean": 88.70796210503077,1179 "ci_low": 88.70796210503077,1180 "ci_high": 88.70796210503077,1181 "n_items": 6,1182 "n_groups": 11183 },1184 "edit_sim": {1185 "mean": 0.6713665943600868,1186 "ci_low": 0.6713665943600868,1187 "ci_high": 0.6713665943600868,1188 "n_items": 6,1189 "n_groups": 11190 },1191 "rougeL": {1192 "mean": 0.7715740451117368,1193 "ci_low": 0.7715740451117368,1194 "ci_high": 0.7715740451117368,1195 "n_items": 6,1196 "n_groups": 11197 },1198 "exact_match": {1199 "mean": 0.0,1200 "ci_low": 0.0,1201 "ci_high": 0.0,1202 "n_items": 6,1203 "n_groups": 11204 },1205 "number_recall": {1206 "mean": 1.0,1207 "ci_low": 1.0,1208 "ci_high": 1.0,1209 "n_items": 6,1210 "n_groups": 11211 },1212 "placeholders_exact": {1213 "mean": 1.0,1214 "ci_low": 1.0,1215 "ci_high": 1.0,1216 "n_items": 6,1217 "n_groups": 11218 }1219 }1220 }1221 },1222 "htr-m1-nano": {1223 "overall": {1224 "n": 26,1225 "chrf": {1226 "mean": 3.9415546328782414,1227 "ci_low": 1.2498295544227394,1228 "ci_high": 8.53519147860751,1229 "n_items": 26,1230 "n_groups": 41231 },1232 "edit_sim": {1233 "mean": 0.12096550612327979,1234 "ci_low": 0.0549310742993309,1235 "ci_high": 0.22563929887977074,1236 "n_items": 26,1237 "n_groups": 41238 },1239 "rougeL": {1240 "mean": 0.028940171413296276,1241 "ci_low": 0.006562385253726958,1242 "ci_high": 0.0749100223482895,1243 "n_items": 26,1244 "n_groups": 41245 },1246 "exact_match": {1247 "mean": 0.0,1248 "ci_low": 0.0,1249 "ci_high": 0.0,1250 "n_items": 26,1251 "n_groups": 41252 },1253 "number_recall": {1254 "mean": 0.19230769230769232,1255 "ci_low": 0.0,1256 "ci_high": 0.38461538461538464,1257 "n_items": 26,1258 "n_groups": 41259 },1260 "placeholders_exact": {1261 "mean": 0.6923076923076923,1262 "ci_low": 0.4,1263 "ci_high": 1.0,1264 "n_items": 26,1265 "n_groups": 41266 }1267 },1268 "by_variant": {1269 "A": {1270 "n": 5,1271 "chrf": {1272 "mean": 3.2142011914105475,1273 "ci_low": 1.134598440262913,1274 "ci_high": 8.231316640801243,1275 "n_items": 5,1276 "n_groups": 41277 },1278 "edit_sim": {1279 "mean": 0.11138352101565976,1280 "ci_low": 0.054622476859231016,1281 "ci_high": 0.20406136698136915,1282 "n_items": 5,1283 "n_groups": 41284 },1285 "rougeL": {1286 "mean": 0.020060449185861546,1287 "ci_low": 0.0010700909577314073,1288 "ci_high": 0.06725576574242031,1289 "n_items": 5,1290 "n_groups": 41291 },1292 "exact_match": {1293 "mean": 0.0,1294 "ci_low": 0.0,1295 "ci_high": 0.0,1296 "n_items": 5,1297 "n_groups": 41298 },1299 "number_recall": {1300 "mean": 0.2,1301 "ci_low": 0.0,1302 "ci_high": 0.42857142857142855,1303 "n_items": 5,1304 "n_groups": 41305 },1306 "placeholders_exact": {1307 "mean": 0.6,1308 "ci_low": 0.25,1309 "ci_high": 1.0,1310 "n_items": 5,1311 "n_groups": 41312 }1313 },1314 "B": {1315 "n": 4,1316 "chrf": {1317 "mean": 3.675383243991121,1318 "ci_low": 0.8846909362721515,1319 "ci_high": 11.307956372076665,1320 "n_items": 4,1321 "n_groups": 31322 },1323 "edit_sim": {1324 "mean": 0.11041130208886771,1325 "ci_low": 0.039887640449438155,1326 "ci_high": 0.2777777777777778,1327 "n_items": 4,1328 "n_groups": 31329 },1330 "rougeL": {1331 "mean": 0.026986814427171518,1332 "ci_low": 0.0,1333 "ci_high": 0.10045662100456622,1334 "n_items": 4,1335 "n_groups": 31336 },1337 "exact_match": {1338 "mean": 0.0,1339 "ci_low": 0.0,1340 "ci_high": 0.0,1341 "n_items": 4,1342 "n_groups": 31343 },1344 "number_recall": {1345 "mean": 0.25,1346 "ci_low": 0.0,1347 "ci_high": 0.5,1348 "n_items": 4,1349 "n_groups": 31350 },1351 "placeholders_exact": {1352 "mean": 0.75,1353 "ci_low": 0.5,1354 "ci_high": 1.0,1355 "n_items": 4,1356 "n_groups": 31357 }1358 },1359 "C": {1360 "n": 4,1361 "chrf": {1362 "mean": 3.685598049744274,1363 "ci_low": 0.8846909362721515,1364 "ci_high": 11.340894021379006,1365 "n_items": 4,1366 "n_groups": 31367 },1368 "edit_sim": {1369 "mean": 0.10793114335870899,1370 "ci_low": 0.039887640449438155,1371 "ci_high": 0.2678571428571429,1372 "n_items": 4,1373 "n_groups": 31374 },1375 "rougeL": {1376 "mean": 0.027335622138992925,1377 "ci_low": 0.0,1378 "ci_high": 0.10185185185185185,1379 "n_items": 4,1380 "n_groups": 31381 },1382 "exact_match": {1383 "mean": 0.0,1384 "ci_low": 0.0,1385 "ci_high": 0.0,1386 "n_items": 4,1387 "n_groups": 31388 },1389 "number_recall": {1390 "mean": 0.25,1391 "ci_low": 0.0,1392 "ci_high": 0.5,1393 "n_items": 4,1394 "n_groups": 31395 },1396 "placeholders_exact": {1397 "mean": 0.75,1398 "ci_low": 0.5,1399 "ci_high": 1.0,1400 "n_items": 4,1401 "n_groups": 31402 }1403 },1404 "D": {1405 "n": 4,1406 "chrf": {1407 "mean": 3.6743235792919164,1408 "ci_low": 0.8846909362721515,1409 "ci_high": 11.295796139569573,1410 "n_items": 4,1411 "n_groups": 31412 },1413 "edit_sim": {1414 "mean": 0.10854498264442325,1415 "ci_low": 0.039887640449438155,1416 "ci_high": 0.27031249999999996,1417 "n_items": 4,1418 "n_groups": 31419 },1420 "rougeL": {1421 "mean": 0.02664743395080474,1422 "ci_low": 0.0,1423 "ci_high": 0.0990990990990991,1424 "n_items": 4,1425 "n_groups": 31426 },1427 "exact_match": {1428 "mean": 0.0,1429 "ci_low": 0.0,1430 "ci_high": 0.0,1431 "n_items": 4,1432 "n_groups": 31433 },1434 "number_recall": {1435 "mean": 0.25,1436 "ci_low": 0.0,1437 "ci_high": 0.5,1438 "n_items": 4,1439 "n_groups": 31440 },1441 "placeholders_exact": {1442 "mean": 0.75,1443 "ci_low": 0.5,1444 "ci_high": 1.0,1445 "n_items": 4,1446 "n_groups": 31447 }1448 },1449 "E": {1450 "n": 9,1451 "chrf": {1452 "mean": 4.696466111741366,1453 "ci_low": 1.4306708710240814,1454 "ci_high": 8.701904531811351,1455 "n_items": 9,1456 "n_groups": 41457 },1458 "edit_sim": {1459 "mean": 0.14229287130655296,1460 "ci_low": 0.06871447875270206,1461 "ci_high": 0.22330958713994528,1462 "n_items": 9,1463 "n_groups": 41464 },1465 "rougeL": {1466 "mean": 0.03647363652761317,1467 "ci_low": 0.006098259528530514,1468 "ci_high": 0.07491726163239712,1469 "n_items": 9,1470 "n_groups": 41471 },1472 "exact_match": {1473 "mean": 0.0,1474 "ci_low": 0.0,1475 "ci_high": 0.0,1476 "n_items": 9,1477 "n_groups": 41478 },1479 "number_recall": {1480 "mean": 0.1111111111111111,1481 "ci_low": 0.0,1482 "ci_high": 0.2727272727272727,1483 "n_items": 9,1484 "n_groups": 41485 },1486 "placeholders_exact": {1487 "mean": 0.6666666666666666,1488 "ci_low": 0.25,1489 "ci_high": 1.0,1490 "n_items": 9,1491 "n_groups": 41492 }1493 }1494 },1495 "by_discipline": {1496 "biomedicine": {1497 "n": 26,1498 "chrf": {1499 "mean": 3.9415546328782414,1500 "ci_low": 1.2498295544227394,1501 "ci_high": 8.53519147860751,1502 "n_items": 26,1503 "n_groups": 41504 },1505 "edit_sim": {1506 "mean": 0.12096550612327979,1507 "ci_low": 0.0549310742993309,1508 "ci_high": 0.22563929887977074,1509 "n_items": 26,1510 "n_groups": 41511 },1512 "rougeL": {1513 "mean": 0.028940171413296276,1514 "ci_low": 0.006562385253726958,1515 "ci_high": 0.0749100223482895,1516 "n_items": 26,1517 "n_groups": 41518 },1519 "exact_match": {1520 "mean": 0.0,1521 "ci_low": 0.0,1522 "ci_high": 0.0,1523 "n_items": 26,1524 "n_groups": 41525 },1526 "number_recall": {1527 "mean": 0.19230769230769232,1528 "ci_low": 0.0,1529 "ci_high": 0.38461538461538464,1530 "n_items": 26,1531 "n_groups": 41532 },1533 "placeholders_exact": {1534 "mean": 0.6923076923076923,1535 "ci_low": 0.4,1536 "ci_high": 1.0,1537 "n_items": 26,1538 "n_groups": 41539 }1540 }1541 },1542 "by_era": {1543 "2000-2009": {1544 "n": 17,1545 "chrf": {1546 "mean": 1.7663500074321619,1547 "ci_low": 0.8846909362721517,1548 "ci_high": 2.2472549553376227,1549 "n_items": 17,1550 "n_groups": 21551 },1552 "edit_sim": {1553 "mean": 0.06554996878160813,1554 "ci_low": 0.039887640449438155,1555 "ci_high": 0.07954760241733717,1556 "n_items": 17,1557 "n_groups": 21558 },1559 "rougeL": {1560 "mean": 0.009060464959341362,1561 "ci_low": 0.007490636704119852,1562 "ci_high": 0.009916734916734917,1563 "n_items": 17,1564 "n_groups": 21565 },1566 "exact_match": {1567 "mean": 0.0,1568 "ci_low": 0.0,1569 "ci_high": 0.0,1570 "n_items": 17,1571 "n_groups": 21572 },1573 "number_recall": {1574 "mean": 0.29411764705882354,1575 "ci_low": 0.0,1576 "ci_high": 0.45454545454545453,1577 "n_items": 17,1578 "n_groups": 21579 },1580 "placeholders_exact": {1581 "mean": 0.7058823529411765,1582 "ci_low": 0.5454545454545454,1583 "ci_high": 1.0,1584 "n_items": 17,1585 "n_groups": 21586 }1587 },1588 "2010-2017": {1589 "n": 9,1590 "chrf": {1591 "mean": 8.050274480943058,1592 "ci_low": 1.9801067907239156,1593 "ci_high": 11.085358326052628,1594 "n_items": 9,1595 "n_groups": 21596 },1597 "edit_sim": {1598 "mean": 0.22563929887977074,1599 "ci_low": 0.13748657357679916,1600 "ci_high": 0.2697156615312566,1601 "n_items": 9,1602 "n_groups": 21603 },1604 "rougeL": {1605 "mean": 0.06649072804854443,1606 "ci_low": 0.004705882352941177,1607 "ci_high": 0.09738315089634604,1608 "n_items": 9,1609 "n_groups": 21610 },1611 "exact_match": {1612 "mean": 0.0,1613 "ci_low": 0.0,1614 "ci_high": 0.0,1615 "n_items": 9,1616 "n_groups": 21617 },1618 "number_recall": {1619 "mean": 0.0,1620 "ci_low": 0.0,1621 "ci_high": 0.0,1622 "n_items": 9,1623 "n_groups": 21624 },1625 "placeholders_exact": {1626 "mean": 0.6666666666666666,1627 "ci_low": 0.0,1628 "ci_high": 1.0,1629 "n_items": 9,1630 "n_groups": 21631 }1632 }1633 },1634 "by_role": {1635 "abstract": {1636 "n": 6,1637 "chrf": {1638 "mean": 0.8846909362721515,1639 "ci_low": 0.8846909362721515,1640 "ci_high": 0.8846909362721515,1641 "n_items": 6,1642 "n_groups": 11643 },1644 "edit_sim": {1645 "mean": 0.039887640449438155,1646 "ci_low": 0.039887640449438155,1647 "ci_high": 0.039887640449438155,1648 "n_items": 6,1649 "n_groups": 11650 },1651 "rougeL": {1652 "mean": 0.007490636704119852,1653 "ci_low": 0.007490636704119852,1654 "ci_high": 0.007490636704119852,1655 "n_items": 6,1656 "n_groups": 11657 },1658 "exact_match": {1659 "mean": 0.0,1660 "ci_low": 0.0,1661 "ci_high": 0.0,1662 "n_items": 6,1663 "n_groups": 11664 },1665 "number_recall": {1666 "mean": 0.0,1667 "ci_low": 0.0,1668 "ci_high": 0.0,1669 "n_items": 6,1670 "n_groups": 11671 },1672 "placeholders_exact": {1673 "mean": 1.0,1674 "ci_low": 1.0,1675 "ci_high": 1.0,1676 "n_items": 6,1677 "n_groups": 11678 }1679 },1680 "data": {1681 "n": 9,1682 "chrf": {1683 "mean": 2.798498176035701,1684 "ci_low": 1.9801067907239156,1685 "ci_high": 3.207693868691594,1686 "n_items": 9,1687 "n_groups": 21688 },1689 "edit_sim": {1690 "mean": 0.11360256737302793,1691 "ci_low": 0.1016605642711423,1692 "ci_high": 0.13748657357679916,1693 "n_items": 9,1694 "n_groups": 21695 },1696 "rougeL": {1697 "mean": 0.013689081238100845,1698 "ci_low": 0.004705882352941177,1699 "ci_high": 0.018180680680680683,1700 "n_items": 9,1701 "n_groups": 21702 },1703 "exact_match": {1704 "mean": 0.0,1705 "ci_low": 0.0,1706 "ci_high": 0.0,1707 "n_items": 9,1708 "n_groups": 21709 },1710 "number_recall": {1711 "mean": 0.0,1712 "ci_low": 0.0,1713 "ci_high": 0.0,1714 "n_items": 9,1715 "n_groups": 21716 },1717 "placeholders_exact": {1718 "mean": 0.6666666666666666,1719 "ci_low": 0.0,1720 "ci_high": 1.0,1721 "n_items": 9,1722 "n_groups": 21723 }1724 },1725 "discussion": {1726 "n": 5,1727 "chrf": {1728 "mean": 1.0947282593128562,1729 "ci_low": 1.0947282593128562,1730 "ci_high": 1.0947282593128562,1731 "n_items": 5,1732 "n_groups": 11733 },1734 "edit_sim": {1735 "mean": 0.053012048192771055,1736 "ci_low": 0.053012048192771055,1737 "ci_high": 0.053012048192771055,1738 "n_items": 5,1739 "n_groups": 11740 },1741 "rougeL": {1742 "mean": 0.0,1743 "ci_low": 0.0,1744 "ci_high": 0.0,1745 "n_items": 5,1746 "n_groups": 11747 },1748 "exact_match": {1749 "mean": 0.0,1750 "ci_low": 0.0,1751 "ci_high": 0.0,1752 "n_items": 5,1753 "n_groups": 11754 },1755 "number_recall": {1756 "mean": 1.0,1757 "ci_low": 1.0,1758 "ci_high": 1.0,1759 "n_items": 5,1760 "n_groups": 11761 },1762 "placeholders_exact": {1763 "mean": 0.0,1764 "ci_low": 0.0,1765 "ci_high": 0.0,1766 "n_items": 5,1767 "n_groups": 11768 }1769 },1770 "related_work": {1771 "n": 6,1772 "chrf": {1773 "mean": 11.08535832605263,1774 "ci_low": 11.08535832605263,1775 "ci_high": 11.08535832605263,1776 "n_items": 6,1777 "n_groups": 11778 },1779 "edit_sim": {1780 "mean": 0.26971566153125665,1781 "ci_low": 0.26971566153125665,1782 "ci_high": 0.26971566153125665,1783 "n_items": 6,1784 "n_groups": 11785 },1786 "rougeL": {1787 "mean": 0.09738315089634604,1788 "ci_low": 0.09738315089634604,1789 "ci_high": 0.09738315089634604,1790 "n_items": 6,1791 "n_groups": 11792 },1793 "exact_match": {1794 "mean": 0.0,1795 "ci_low": 0.0,1796 "ci_high": 0.0,1797 "n_items": 6,1798 "n_groups": 11799 },1800 "number_recall": {1801 "mean": 0.0,1802 "ci_low": 0.0,1803 "ci_high": 0.0,1804 "n_items": 6,1805 "n_groups": 11806 },1807 "placeholders_exact": {1808 "mean": 1.0,1809 "ci_low": 1.0,1810 "ci_high": 1.0,1811 "n_items": 6,1812 "n_groups": 11813 }1814 }1815 }1816 }1817 },1818 "paired_vs_baseline": {1819 "B1_generic_paraphrase": {1820 "chrf": {1821 "mean": -1.544733717352225,1822 "ci_low": -2.081107341699917,1823 "ci_high": -0.6362969154408691,1824 "n_items": 26,1825 "n_groups": 4,1826 "p_value_two_sided": 0.01827 },1828 "edit_sim": {1829 "mean": -0.0790762877380403,1830 "ci_low": -0.10670083365283885,1831 "ci_high": -0.039249371010325836,1832 "n_items": 26,1833 "n_groups": 4,1834 "p_value_two_sided": 0.01835 },1836 "rougeL": {1837 "mean": -0.054019592002711454,1838 "ci_low": -0.06856345335967809,1839 "ci_high": -0.02908739758771804,1840 "n_items": 26,1841 "n_groups": 4,1842 "p_value_two_sided": 0.01843 },1844 "exact_match": {1845 "mean": 0.0,1846 "ci_low": 0.0,1847 "ci_high": 0.0,1848 "n_items": 26,1849 "n_groups": 4,1850 "p_value_two_sided": 0.01851 },1852 "number_recall": {1853 "mean": 0.0,1854 "ci_low": 0.0,1855 "ci_high": 0.0,1856 "n_items": 26,1857 "n_groups": 4,1858 "p_value_two_sided": 0.01859 },1860 "placeholders_exact": {1861 "mean": 0.0,1862 "ci_low": 0.0,1863 "ci_high": 0.0,1864 "n_items": 26,1865 "n_groups": 4,1866 "p_value_two_sided": 0.01867 }1868 },1869 "htr-m1-nano": {1870 "chrf": {1871 "mean": -87.19889145130885,1872 "ci_low": -91.36412190208351,1873 "ci_high": -82.52507723239988,1874 "n_items": 26,1875 "n_groups": 4,1876 "p_value_two_sided": 0.01877 },1878 "edit_sim": {1879 "mean": -0.7239390729066955,1880 "ci_low": -0.8210444712968841,1881 "ci_high": -0.5836619962181375,1882 "n_items": 26,1883 "n_groups": 4,1884 "p_value_two_sided": 0.01885 },1886 "rougeL": {1887 "mean": -0.8503140683701882,1888 "ci_low": -0.8930333250420825,1889 "ci_high": -0.7854963372502638,1890 "n_items": 26,1891 "n_groups": 4,1892 "p_value_two_sided": 0.01893 },1894 "exact_match": {1895 "mean": 0.0,1896 "ci_low": 0.0,1897 "ci_high": 0.0,1898 "n_items": 26,1899 "n_groups": 4,1900 "p_value_two_sided": 0.01901 },1902 "number_recall": {1903 "mean": -0.8076923076923077,1904 "ci_low": -1.0,1905 "ci_high": -0.6153846153846154,1906 "n_items": 26,1907 "n_groups": 4,1908 "p_value_two_sided": 0.01909 },1910 "placeholders_exact": {1911 "mean": -0.3076923076923077,1912 "ci_low": -0.6,1913 "ci_high": 0.0,1914 "n_items": 26,1915 "n_groups": 4,1916 "p_value_two_sided": 0.01917 }1918 }1919 },1920 "stylometry": {1921 "rewrite_vs_original": {1922 "W1_mean_sent_len": 0.5353909380473728,1923 "W1_sd_sent_len": 0.5242467850892425,1924 "W1_ttr": 0.05332844778286766,1925 "W1_mtld": 0.22043371369496564,1926 "W1_mean_word_len": 0.49584966685247417,1927 "W1_long_word_rate": 0.3288194224634678,1928 "W1_comma_rate": 0.06311357929865084,1929 "W1_semicolon_rate": 1095173.220385682,1930 "W1_paren_rate": 0.01033112052339397,1931 "W1_discourse_rate": 0.11767701280789468,1932 "W1_passive_rate": 0.32293510751265214,1933 "W1_nominal_rate": 0.4026044998131327,1934 "W1_first_person_rate": 0.007195982589714723,1935 "W1_subord_rate": 0.334448210443354,1936 "W1_hedge_rate": 0.15981859771643503,1937 "W1_negation_rate": 0.0033111035811765146,1938 "W1_digit_rate": 0.031022947377273694,1939 "W1_flesch": 0.29746440502364396,1940 "W1_mean": 60843.1737987346061941 },1942 "feature_means": {1943 "original": {1944 "mean_sent_len": 20.39102564102564,1945 "sd_sent_len": 7.13049261583026,1946 "ttr": 0.6809402896052072,1947 "mtld": 62.51055519541926,1948 "mean_word_len": 5.7285335377352045,1949 "long_word_rate": 0.12795968607908087,1950 "comma_rate": 0.04997363134812845,1951 "semicolon_rate": 0.0,1952 "paren_rate": 0.020013568521031207,1953 "discourse_rate": 0.007912057871051857,1954 "passive_rate": 0.5961538461538463,1955 "nominal_rate": 0.06501094606494343,1956 "first_person_rate": 0.001791600311099589,1957 "subord_rate": 0.024452584047991374,1958 "hedge_rate": 0.010708878327795768,1959 "negation_rate": 0.009778357235984355,1960 "digit_rate": 0.012007462807683581,1961 "flesch": 18.243351656690741962 },1963 "rewrite": {1964 "mean_sent_len": 22.70149572649573,1965 "sd_sent_len": 9.073090685458082,1966 "ttr": 0.6752771076177139,1967 "mtld": 63.67492163149923,1968 "mean_word_len": 5.800808707752291,1969 "long_word_rate": 0.13238095487623885,1970 "comma_rate": 0.05126429731383023,1971 "semicolon_rate": 0.0013171921689599806,1972 "paren_rate": 0.020269032058641702,1973 "discourse_rate": 0.007517255279717731,1974 "passive_rate": 0.6657509157509158,1975 "nominal_rate": 0.07298796138305398,1976 "first_person_rate": 0.0017732440448388269,1977 "subord_rate": 0.018481951518139854,1978 "hedge_rate": 0.007388322732661601,1979 "negation_rate": 0.009714236860666425,1980 "digit_rate": 0.011890363662258886,1981 "flesch": 14.7447813744358781982 }1983 },1984 "B0_identity": {1985 "vs_original": {1986 "W1_mean_sent_len": 0.5353909380473728,1987 "W1_sd_sent_len": 0.5242467850892425,1988 "W1_ttr": 0.05332844778286766,1989 "W1_mtld": 0.22043371369496564,1990 "W1_mean_word_len": 0.49584966685247417,1991 "W1_long_word_rate": 0.3288194224634678,1992 "W1_comma_rate": 0.06311357929865084,1993 "W1_semicolon_rate": 1095173.220385682,1994 "W1_paren_rate": 0.01033112052339397,1995 "W1_discourse_rate": 0.11767701280789468,1996 "W1_passive_rate": 0.32293510751265214,1997 "W1_nominal_rate": 0.4026044998131327,1998 "W1_first_person_rate": 0.007195982589714723,1999 "W1_subord_rate": 0.334448210443354,2000 "W1_hedge_rate": 0.15981859771643503,2001 "W1_negation_rate": 0.0033111035811765146,2002 "W1_digit_rate": 0.031022947377273694,2003 "W1_flesch": 0.29746440502364396,2004 "W1_mean": 60843.1737987346062005 },2006 "feature_means": {2007 "mean_sent_len": 22.70149572649573,2008 "sd_sent_len": 9.073090685458082,2009 "ttr": 0.6752771076177139,2010 "mtld": 63.67492163149923,2011 "mean_word_len": 5.800808707752291,2012 "long_word_rate": 0.13238095487623885,2013 "comma_rate": 0.05126429731383023,2014 "semicolon_rate": 0.0013171921689599806,2015 "paren_rate": 0.020269032058641702,2016 "discourse_rate": 0.007517255279717731,2017 "passive_rate": 0.6657509157509158,2018 "nominal_rate": 0.07298796138305398,2019 "first_person_rate": 0.0017732440448388269,2020 "subord_rate": 0.018481951518139854,2021 "hedge_rate": 0.007388322732661601,2022 "negation_rate": 0.009714236860666425,2023 "digit_rate": 0.011890363662258886,2024 "flesch": 14.7447813744358782025 }2026 },2027 "B1_generic_paraphrase": {2028 "vs_original": {2029 "W1_mean_sent_len": 0.7924740565894703,2030 "W1_sd_sent_len": 0.891518367957381,2031 "W1_ttr": 0.06615956651272022,2032 "W1_mtld": 0.282269735470249,2033 "W1_mean_word_len": 0.5827060799467206,2034 "W1_long_word_rate": 0.40547903342866887,2035 "W1_comma_rate": 0.062087228949551966,2036 "W1_semicolon_rate": 1095173.220385682,2037 "W1_paren_rate": 0.009644498069968813,2038 "W1_discourse_rate": 0.11767701280789468,2039 "W1_passive_rate": 0.49722952301597606,2040 "W1_nominal_rate": 0.5086244683466113,2041 "W1_first_person_rate": 0.007195982589714723,2042 "W1_subord_rate": 0.3881380134986631,2043 "W1_hedge_rate": 0.15981859771643503,2044 "W1_negation_rate": 0.0033111035811765146,2045 "W1_digit_rate": 0.035910159378172817,2046 "W1_flesch": 0.40869574714412377,2047 "W1_mean": 60843.246629158732048 },2049 "feature_means": {2050 "mean_sent_len": 23.948290598290598,2051 "sd_sent_len": 10.604262223658505,2052 "ttr": 0.6738332776165504,2053 "mtld": 63.694742528175205,2054 "mean_word_len": 5.821524631876246,2055 "long_word_rate": 0.13385798799181703,2056 "comma_rate": 0.05131480292188463,2057 "semicolon_rate": 0.0013171921689599806,2058 "paren_rate": 0.020302702464011304,2059 "discourse_rate": 0.007517255279717731,2060 "passive_rate": 0.7042124542124544,2061 "nominal_rate": 0.07503222455793111,2062 "first_person_rate": 0.0017732440448388269,2063 "subord_rate": 0.01734226349376469,2064 "hedge_rate": 0.007388322732661601,2065 "negation_rate": 0.009714236860666425,2066 "digit_rate": 0.011864264364394294,2067 "flesch": 13.2732465335830822068 }2069 },2070 "htr-m1-nano": {2071 "vs_original": {2072 "W1_mean_sent_len": 10.398642546857173,2073 "W1_sd_sent_len": 1.8179959515548545,2074 "W1_ttr": 2.9460584212758762,2075 "W1_mtld": 5.100685980105192,2076 "W1_mean_word_len": 807.4821604231622,2077 "W1_long_word_rate": 13.689440066228036,2078 "W1_comma_rate": 2.056993220929629,2079 "W1_semicolon_rate": 0.0,2080 "W1_paren_rate": 0.6274525631928255,2081 "W1_discourse_rate": 0.842848467367277,2082 "W1_passive_rate": 2.845488374315631,2083 "W1_nominal_rate": 3.313723045447269,2084 "W1_first_person_rate": 0.65311190077176,2085 "W1_subord_rate": 1.39144862458145,2086 "W1_hedge_rate": 0.5187982620068583,2087 "W1_negation_rate": 0.46076075771809305,2088 "W1_digit_rate": 1.907620087182116,2089 "W1_flesch": 10.574415888433773,2090 "W1_mean": 48.145980254507222091 },2092 "feature_means": {2093 "mean_sent_len": 49.57692307692308,2094 "sd_sent_len": 0.0,2095 "ttr": 0.6714946184208446,2096 "mtld": 3.667780972572053,2097 "mean_word_len": 136.71176492744306,2098 "long_word_rate": 0.4294871794871796,2099 "comma_rate": 0.051509846678744885,2100 "semicolon_rate": 0.0,2101 "paren_rate": 0.002903791365329827,2102 "discourse_rate": 0.0,2103 "passive_rate": 0.0,2104 "nominal_rate": 0.0,2105 "first_person_rate": 0.0,2106 "subord_rate": 0.007839294463498285,2107 "hedge_rate": 0.0,2108 "negation_rate": 0.0,2109 "digit_rate": 0.0,2110 "flesch": -7.1821386785706192111 }2112 }2113 }2114}2115