{
  "document_id": "lb_qasper_92da01e7242f30f5266e431b4269fee1b0ca5fcc23aee095",
  "family": "public_qasper",
  "split": "test",
  "document_hash": "9d5b2acc8346d6e1f64cc3c76c1c46b2edad499c17909fa18f8c1a4c20b844d7",
  "questions": [
    {
      "question_id": "q00",
      "qtype": "qasper",
      "question": "How are models evaluated in this human-machine communication game?",
      "answer": "by training an autocomplete system on 500K randomly sampled sentences from Yelp reviews",
      "aliases": [
        "efficiency of a communication scheme $(q_{\\alpha },p_{\\beta })$ by the retention rate of tokens, which is measured as the fraction of tokens that are kept in the keywords, accuracy of a scheme is measured as the fraction of sentences generated by greedily decoding the model that exactly matches the target sentence"
      ],
      "answer_type": "qa_f1",
      "tolerance": 0.0,
      "supporting_fact_ids": [],
      "answer_position_frac": -1.0
    }
  ],
  "facts": []
}