1{2 "document_id": "lb_qasper_92da01e7242f30f5266e431b4269fee1b0ca5fcc23aee095",3 "family": "public_qasper",4 "split": "test",5 "document_hash": "9d5b2acc8346d6e1f64cc3c76c1c46b2edad499c17909fa18f8c1a4c20b844d7",6 "questions": [7 {8 "question_id": "q00",9 "qtype": "qasper",10 "question": "How are models evaluated in this human-machine communication game?",11 "answer": "by training an autocomplete system on 500K randomly sampled sentences from Yelp reviews",12 "aliases": [13 "efficiency of a communication scheme $(q_{\\alpha },p_{\\beta })$ by the retention rate of tokens, which is measured as the fraction of tokens that are kept in the keywords, accuracy of a scheme is measured as the fraction of sentences generated by greedily decoding the model that exactly matches the target sentence"14 ],15 "answer_type": "qa_f1",16 "tolerance": 0.0,17 "supporting_fact_ids": [],18 "answer_position_frac": -1.019 }20 ],21 "facts": []22}