{
  "document_id": "lb_qasper_a8f155f6d0b86040e21fdb913b07c5fa675b82dbacc0674e",
  "family": "public_qasper",
  "split": "test",
  "document_hash": "a8d3111f6fcf690198f2a5583b971a40f98811906d21a4651a641d840a3d37cc",
  "questions": [
    {
      "question_id": "q00",
      "qtype": "qasper",
      "question": "How much improvement is gained from Adversarial Reward Augmented Maximum Likelihood (ARAML)?",
      "answer": "ARAM has achieved improvement over all baseline methods using reverese perplexity and  slef-BLEU metric.  The maximum reverse perplexity improvement 936,16 is gained for EMNLP2017  WMT  dataset and  48,44 for COCO dataset.",
      "aliases": [
        "Compared to the baselines, ARAML does not do better in terms of perplexity on COCO and EMNLP 2017 WMT datasets, but it does by up to 0.27 Self-BLEU points on COCO and 0.35 Self-BLEU on EMNLP 2017 WMT. In terms of Grammaticality and Relevance, it scores better than the baselines on up to 75.5% and 73% of the cases respectively."
      ],
      "answer_type": "qa_f1",
      "tolerance": 0.0,
      "supporting_fact_ids": [],
      "answer_position_frac": -1.0
    }
  ],
  "facts": []
}