1{2 "document_id": "lb_qasper_a8f155f6d0b86040e21fdb913b07c5fa675b82dbacc0674e",3 "family": "public_qasper",4 "split": "test",5 "document_hash": "a8d3111f6fcf690198f2a5583b971a40f98811906d21a4651a641d840a3d37cc",6 "questions": [7 {8 "question_id": "q00",9 "qtype": "qasper",10 "question": "How much improvement is gained from Adversarial Reward Augmented Maximum Likelihood (ARAML)?",11 "answer": "ARAM has achieved improvement over all baseline methods using reverese perplexity and slef-BLEU metric. The maximum reverse perplexity improvement 936,16 is gained for EMNLP2017 WMT dataset and 48,44 for COCO dataset.",12 "aliases": [13 "Compared to the baselines, ARAML does not do better in terms of perplexity on COCO and EMNLP 2017 WMT datasets, but it does by up to 0.27 Self-BLEU points on COCO and 0.35 Self-BLEU on EMNLP 2017 WMT. In terms of Grammaticality and Relevance, it scores better than the baselines on up to 75.5% and 73% of the cases respectively."14 ],15 "answer_type": "qa_f1",16 "tolerance": 0.0,17 "supporting_fact_ids": [],18 "answer_position_frac": -1.019 }20 ],21 "facts": []22}