spb/wp20_uqo

1{"status": "ok", "message-type": "work", "message-version": "1.0.0", "message": {"indexed": {"date-parts": [[2026, 10, 20]], "date-time": "2026-10-20T03:02:13Z", "timestamp": 1792465333137, "version": "4.5.0"}, "reference-count": 51, "publisher": "MIT Press", "issue": "4", "content-domain": {"domain": [], "crossmark-restriction": false}, "short-container-title": ["Computational Linguistics"], "published-print": {"date-parts": [[2008, 12]]}, "abstract": "<jats:p>This article is a survey of methods for measuring agreement among corpus annotators. It exposes the mathematics and underlying assumptions of agreement coefficients, covering Krippendorff's alpha as well as Scott's pi and Cohen's kappa; discusses the use of coefficients in several annotation tasks; and argues that weighted, alpha-like coefficients, traditionally less used than kappa-like measures in computational linguistics, may be more appropriate for many corpus annotation tasks—but that their use makes the interpretation of the value of the coefficient even harder.</jats:p>", "DOI": "10.1162/coli.07-034-r2", "type": "journal-article", "created": {"date-parts": [[2008, 9, 30]], "date-time": "2008-09-30T13:13:56Z", "timestamp": 1222780436000}, "page": "555-596", "source": "Crossref", "is-referenced-by-count": 981, "title": ["Inter-Coder Agreement for Computational Linguistics"], "prefix": "10.1162", "volume": "34", "author": [{"given": "Ron", "family": "Artstein", "sequence": "first", "affiliation": [{"name": "* Now at the Institute for Creative Technologies, University of Southern California, 13274 Fiji Way, Marina Del Rey, CA 90292."}, {"name": "** At the University of Essex: Department of Computing and Electronic Systems, University of Essex, Wivenhoe Park, Colchester, CO4 3SQ, UK.. At the University of Trento: CIMeC, Università degli Studi di Trento, Palazzo Fedrigotti, Corso Bettini, 31, 38068 Rovereto (TN), Italy.."}], "role": [{"vocabulary": "crossref", "role": "author"}]}, {"given": "Massimo", "family": "Poesio", "sequence": "additional", "affiliation": [{"name": "* Now at the Institute for Creative Technologies, University of Southern California, 13274 Fiji Way, Marina Del Rey, CA 90292."}, {"name": "** At the University of Essex: Department of Computing and Electronic Systems, University of Essex, Wivenhoe Park, Colchester, CO4 3SQ, UK.. At the University of Trento: CIMeC, Università degli Studi di Trento, Palazzo Fedrigotti, Corso Bettini, 31, 38068 Rovereto (TN), Italy.."}], "role": [{"vocabulary": "crossref", "role": "author"}]}], "member": "281", "reference": [{"key": "p_4", "first-page": "5", "volume": "41", "author": "Atkins Sue", "year": "1992", "journal-title": "Acta Linguistica Hungarica"}, {"key": "p_5", "doi-asserted-by": "publisher", "DOI": "10.1017/S1351324905003803"}, {"key": "p_6", "doi-asserted-by": "publisher", "DOI": "10.1097/00005053-197611000-00003"}, {"key": "p_7", "doi-asserted-by": "publisher", "DOI": "10.1023/A:1007506220214"}, {"key": "p_8", "doi-asserted-by": "publisher", "DOI": "10.1086/266520"}, {"key": "p_9", "doi-asserted-by": "publisher", "DOI": "10.2307/2532052"}, {"key": "p_10", "doi-asserted-by": "publisher", "DOI": "10.1177/001316448104100307"}, {"key": "p_12", "doi-asserted-by": "publisher", "DOI": "10.1017/S1351324999002181"}, {"key": "p_17", "doi-asserted-by": "publisher", "DOI": "10.1016/0895-4356(93)90018-V"}, {"issue": "2", "key": "p_18", "first-page": "249", "volume": "22", "author": "Carletta Jean", "year": "1996", "journal-title": "Computational Linguistics"}, {"issue": "1", "key": "p_19", "first-page": "13", "volume": "23", "author": "Carletta Jean", "year": "1997", "journal-title": "Computational Linguistics"}, {"key": "p_21", "doi-asserted-by": "publisher", "DOI": "10.1016/0895-4356(90)90159-M"}, {"key": "p_22", "doi-asserted-by": "publisher", "DOI": "10.1177/001316446002000104"}, {"key": "p_23", "doi-asserted-by": "publisher", "DOI": "10.1037/h0026256"}, {"key": "p_26", "doi-asserted-by": "publisher", "DOI": "10.1162/089120105774321109"}, {"key": "p_27", "doi-asserted-by": "publisher", "DOI": "10.2307/2529886"}, {"key": "p_29", "doi-asserted-by": "publisher", "DOI": "10.1162/089120104773633402"}, {"key": "p_31", "doi-asserted-by": "publisher", "DOI": "10.2307/1932409"}, {"key": "p_32", "doi-asserted-by": "publisher", "DOI": "10.1002/sim.4780060404"}, {"key": "p_34", "doi-asserted-by": "publisher", "DOI": "10.1093/jos/17.1.51"}, {"key": "p_35", "doi-asserted-by": "publisher", "DOI": "10.1016/0895-4356(90)90158-L"}, {"key": "p_37", "doi-asserted-by": "publisher", "DOI": "10.1037/h0031619"}, {"key": "p_38", "doi-asserted-by": "publisher", "DOI": "10.2307/2529549"}, {"issue": "3", "key": "p_42", "first-page": "175", "volume": "12", "author": "Grosz Barbara J", "year": "1986", "journal-title": "Computational Linguistics"}, {"key": "p_43", "doi-asserted-by": "publisher", "DOI": "10.1080/19312450709336664"}, {"issue": "1", "key": "p_44", "first-page": "33", "volume": "23", "author": "Hearst Marti A", "year": "1997", "journal-title": "Computational Linguistics"}, {"key": "p_46", "doi-asserted-by": "publisher", "DOI": "10.1207/S15328031US0203_03"}, {"key": "p_47", "doi-asserted-by": "publisher", "DOI": "10.1111/j.1469-8137.1912.tb05611.x"}, {"key": "p_53", "doi-asserted-by": "publisher", "DOI": "10.1177/001316447003000105"}, {"issue": "1", "key": "p_54", "first-page": "142", "volume": "34", "author": "Krippendorff Klaus", "year": "1978", "journal-title": "Biometrics"}, {"key": "p_56", "doi-asserted-by": "publisher", "DOI": "10.2307/271061"}, {"issue": "3", "key": "p_58", "first-page": "411", "volume": "30", "author": "Krippendorff Klaus", "year": "2004", "journal-title": "Human Communication Research"}, {"key": "p_59", "doi-asserted-by": "publisher", "DOI": "10.2307/2529310"}, {"key": "p_61", "doi-asserted-by": "publisher", "DOI": "10.1016/S0364-0213(77)80016-5"}, {"issue": "2", "key": "p_64", "first-page": "313", "volume": "19", "author": "Marcus Mitchell P", "year": "1993", "journal-title": "Computational Linguistics"}, {"key": "p_66", "doi-asserted-by": "publisher", "DOI": "10.1023/A:1002402902356"}, {"key": "p_74", "doi-asserted-by": "publisher", "DOI": "10.1017/S135132490500402X"}, {"issue": "1", "key": "p_80", "first-page": "103", "volume": "23", "author": "Passonneau Rebecca J", "year": "1997", "journal-title": "Computational Linguistics"}, {"key": "p_81", "doi-asserted-by": "publisher", "DOI": "10.1162/089120102317341756"}, {"key": "p_86", "doi-asserted-by": "publisher", "DOI": "10.1007/s11168-006-9005-z"}, {"issue": "2", "key": "p_87", "first-page": "183", "volume": "24", "author": "Poesio Massimo", "year": "1998", "journal-title": "Computational Linguistics"}, {"key": "p_89", "doi-asserted-by": "publisher", "DOI": "10.1002/sim.4780090917"}, {"key": "p_90", "doi-asserted-by": "publisher", "DOI": "10.1007/BF02289730"}, {"key": "p_91", "doi-asserted-by": "publisher", "DOI": "10.1162/coli.2008.34.3.319"}, {"issue": "1", "key": "p_92", "first-page": "53", "volume": "27", "author": "Reinhart T.", "year": "1981", "journal-title": "Philosophica"}, {"key": "p_97", "doi-asserted-by": "publisher", "DOI": "10.1086/266577"}, {"key": "p_102", "doi-asserted-by": "publisher", "DOI": "10.1162/089120100561737"}, {"key": "p_103", "doi-asserted-by": "publisher", "DOI": "10.2307/2333387"}, {"key": "p_105", "doi-asserted-by": "publisher", "DOI": "10.1162/089120102762671936"}, {"key": "p_106", "doi-asserted-by": "publisher", "DOI": "10.1111/j.1467-8640.1992.tb00380.x"}, {"key": "p_110", "doi-asserted-by": "publisher", "DOI": "10.1037/0033-2909.103.3.374"}], "container-title": ["Computational Linguistics"], "original-title": [], "language": "en", "link": [{"URL": "https://www.mitpressjournals.org/doi/pdf/10.1162/coli.07-034-R2", "content-type": "unspecified", "content-version": "vor", "intended-application": "similarity-checking"}], "deposited": {"date-parts": [[2021, 3, 12]], "date-time": "2021-03-12T16:25:23Z", "timestamp": 1615566323000}, "score": 1, "resource": {"primary": {"URL": "https://direct.mit.edu/coli/article/34/4/555-596/1999"}}, "subtitle": [], "short-title": [], "issued": {"date-parts": [[2008, 12]]}, "references-count": 51, "journal-issue": {"issue": "4", "published-print": {"date-parts": [[2008, 12]]}}, "alternative-id": ["10.1162/coli.07-034-R2"], "URL": "https://doi.org/10.1162/coli.07-034-r2", "relation": {}, "ISSN": ["0891-2017", "1530-9312"], "issn-type": [{"value": "0891-2017", "type": "print"}, {"value": "1530-9312", "type": "electronic"}], "subject": [], "published": {"date-parts": [[2008, 12]]}}}