1{2 "document_id": "lb_qasper_86c75d2a9157cb601a49f8424fa1e6e90fcd724132490bbd",3 "family": "public_qasper",4 "split": "test",5 "document_hash": "8f673b48b0058e265bda13f9c98850a581b541a1e33ab67f6dc42bdddf9c135d",6 "questions": [7 {8 "question_id": "q00",9 "qtype": "qasper",10 "question": "What metrics are used for evaluation?",11 "answer": "Byte-Pair Encoding perplexity (BPE PPL),\nBLEU-1,\nBLEU-4,\nROUGE-L,\npercentage of distinct unigram (D-1),\npercentage of distinct bigrams(D-2),\nuser matching accuracy(UMA),\nMean Reciprocal Rank(MRR)\nPairwise preference over baseline(PP)",12 "aliases": [13 "BLEU-1/4 and ROUGE-L, likelihood of generated recipes using identical input specifications but conditioned on ten different user profiles, user matching accuracy (UMA), Mean Reciprocal Rank (MRR), neural scoring model from BIBREF33 to measure recipe-level coherence",14 " Distinct-1/2, UMA = User Matching Accuracy, MRR\n= Mean Reciprocal Rank, PP = Pairwise preference over baseline (evaluated for 310 recipe pairs per model)"15 ],16 "answer_type": "qa_f1",17 "tolerance": 0.0,18 "supporting_fact_ids": [],19 "answer_position_frac": -1.020 }21 ],22 "facts": []23}