Repository Public
1// ORIX Docs2// File: Packages/ORIXDocsKit/Tests/DocReferencesTests/InterchangeTests.swift3// Description: RIS and CSL-JSON parsing and round trips, format detection, library merging and de-duplication, completeness checks4// and key suggestion.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import Testing10import DocModel11@testable import DocReferences1213@Suite struct RISTests {14 static let sample = """15 TY - JOUR16 ID - bollerslev198617 AU - Bollerslev, Tim18 A1 - Engle, Robert F., Jr.19 TI - Generalized autoregressive conditional20 heteroskedasticity21 JO - Journal of Econometrics22 JA - J. Econom.23 PY - 198624 DA - 1986/04/15/25 VL - 3126 IS - 327 SP - 30728 EP - 32729 DO - https://doi.org/10.1016/0304-4076(86)90063-130 UR - https://example.org/garch31 SN - 0304-407632 AB - A natural generalization of the ARCH process.33 KW - GARCH34 KW - volatility35 L1 - file:///papers/garch.pdf36 ER - 3738 TY - CHAP39 AU - World Health Organization40 A2 - Roe, Richard41 T1 - A chapter42 T2 - The Book43 PB - Springer44 CY - Berlin45 Y1 - 2001///Spring46 SP - 12-3447 SN - 978-3-16-148410-048 ET - 2nd49 ER -5051 TY - THES52 AU - Roy, Anne53 TI - A thesis54 PB - Université du Québec en Outaouais55 M3 - PhD thesis56 PY - in press57 Y2 - 2024/02/03/58 TY - ZZZZ59 TI - Unknown type60 """6162 @Test func parsesStandardTags() {63 let r = RISParser.parse(Self.sample)64 #expect(r.entries.count == 4)65 let a = r.entries[0]66 #expect(a.key == "bollerslev1986")67 #expect(a.type == "article-journal")68 #expect(a.authors == [person("Bollerslev", "Tim"), person("Engle", "Robert F., Jr.")])69 #expect(a.title == "Generalized autoregressive conditional heteroskedasticity") // multi-line value70 #expect(a.fields["container-title"] == "Journal of Econometrics")71 #expect(a.fields["container-title-short"] == "J. Econom.")72 #expect([a.year, a.month, a.day] == [1986, 4, 15])73 #expect(a.fields["volume"] == "31")74 #expect(a.fields["issue"] == "3")75 #expect(a.fields["page"] == "307–327")76 #expect(a.fields["DOI"] == "10.1016/0304-4076(86)90063-1")77 #expect(a.fields["URL"] == "https://example.org/garch")78 #expect(a.fields["ISSN"] == "0304-4076")79 #expect(a.fields["abstract"] == "A natural generalization of the ARCH process.")80 #expect(a.fields["keyword"] == "GARCH; volatility")81 #expect(a.fields["L1"] == "file:///papers/garch.pdf") // unmapped tag kept82 let b = r.entries[1]83 #expect(b.type == "chapter")84 #expect(b.authors == [org("World Health Organization")])85 #expect(b.editors == [person("Roe", "Richard")])86 #expect(b.fields["container-title"] == "The Book")87 #expect(b.fields["publisher"] == "Springer")88 #expect(b.fields["publisher-place"] == "Berlin")89 #expect(b.year == 2001)90 #expect(b.month == nil)91 #expect(b.fields["season"] == "Spring")92 #expect(b.fields["page"] == "12–34")93 #expect(b.fields["ISBN"] == "978-3-16-148410-0")94 #expect(b.fields["edition"] == "2nd")95 #expect(b.key == "who2001chapter") // generated: the record has no ID96 let c = r.entries[2]97 #expect(c.type == "thesis")98 #expect(c.year == nil)99 #expect(c.fields[ReferenceField.issuedLiteral] == "in press")100 #expect(c.fields["genre"] == "PhD thesis")101 #expect(c.fields["accessed"] == "2024-02-03")102 #expect(r.entries[3].type == "document")103 #expect(r.entries[3].fields["TY"] == "ZZZZ")104 let zLine = Self.sample.split(separator: "\n", omittingEmptySubsequences: false).firstIndex { $0.contains("ZZZZ") }! + 1105 #expect(r.issues.contains { $0.hasPrefix("Line \(zLine):") && $0.contains("before the previous one") })106 #expect(r.issues.contains { $0.contains("unknown reference type \"ZZZZ\"") })107 #expect(r.issues.contains { $0.contains("not closed with ER") })108 }109110 @Test func roundTrip() {111 let first = RISParser.parse(Self.sample).entries112 let written = RISWriter.write(first)113 let second = RISParser.parse(written)114 #expect(second.issues.count == 1 && second.issues[0].contains("ZZZZ"), "\(second.issues)") // the unknown type is still unknown115 #expect(second.entries == first)116 #expect(written.contains("TY - JOUR\n"))117 #expect(written.contains("SP - 307\nEP - 327\n"))118 #expect(written.contains("TY - ZZZZ\n"))119 // Type mapping both ways for every CSL type the writer knows.120 for (csl, ris) in RISMapping.fromCSL where !["article", "post", "entry"].contains(csl) {121 #expect(RISMapping.toCSL[ris] == csl, "\(csl) → \(ris)")122 let e = BibEntry(key: "k", type: csl, authors: [person("Roy", "Anne"), person("Plato"), org("Statistics Canada"), .others], year: 2020, fields: ["title": "T"])123 #expect(RISParser.parse(RISWriter.write([e])).entries == [e], "\(csl)")124 }125 }126}127128@Suite struct CSLJSONTests {129 static let sample = #"""130 [131 {"id": "boucher2024", "type": "article-journal", "title": "Volatility <i>in vivo</i>",132 "author": [{"family": "Boucher", "given": "Simon-Pierre"}, {"family": "Gogh", "given": "Vincent", "non-dropping-particle": "van"},133 {"family": "Beethoven", "given": "Ludwig", "dropping-particle": "van"}, {"family": "King", "given": "Martin Luther", "suffix": "Jr."},134 {"literal": "Bank of Canada"}],135 "editor": [{"family": "Roe", "given": "R."}],136 "translator": [{"family": "Goldstein", "given": "Ann"}],137 "issued": {"date-parts": [[2024, 3, 15]]}, "accessed": {"date-parts": [["2024", "4", "1"]]},138 "container-title": "Journal of Finance", "volume": 79, "issue": "2", "page": "100-120", "DOI": "https://doi.org/10.1111/jofi.1", "custom": {"a": 1}},139 {"id": 42, "type": "book", "title": "Numbers as strings", "issued": {"date-parts": [["1999"]]}, "edition": 3, "publisher": "P"},140 {"id": "raw", "type": "report", "title": "Raw date", "issued": {"raw": "2020-06-01"}, "number": 12},141 {"id": "lit", "type": "manuscript", "title": "Literal date", "issued": {"literal": "forthcoming"}},142 {"id": "range", "type": "paper-conference", "title": "Range", "issued": {"date-parts": [[2019, 5, 1], [2019, 5, 3]]}, "event-date": {"raw": "2019-05-01/2019-05-03"}},143 {"id": "season", "type": "article-journal", "title": "Season", "issued": {"date-parts": [[2017]], "season": 1}},144 {"id": "str", "type": "book", "title": "String date", "issued": "2001-02"},145 {"type": "book", "title": "No identifier here", "author": [{"family": "Roy", "given": "Anne"}], "issued": {"date-parts": [[2005]]}},146 "not an object"147 ]148 """#149150 @Test func parsesNamesDatesAndLooseTypes() throws {151 let r = CSLJSON.parse(Data(Self.sample.utf8))152 #expect(r.entries.count == 8)153 let a = r.entries[0]154 #expect(a.authors == [person("Boucher", "Simon-Pierre"), person("van Gogh", "Vincent"), person("Beethoven", "Ludwig van"), person("King", "Martin Luther, Jr."),155 org("Bank of Canada")])156 #expect(a.editors == [person("Roe", "R.")])157 #expect(a.fields["translator"] == "Goldstein, Ann")158 #expect([a.year, a.month, a.day] == [2024, 3, 15])159 #expect(a.fields["accessed"] == "2024-04-01")160 #expect(a.fields["volume"] == "79")161 #expect(a.fields["DOI"] == "10.1111/jofi.1")162 #expect(a.fields["custom"] == #"{"a":1}"#)163 #expect(r.entries[1].key == "42")164 #expect(r.entries[1].year == 1999)165 #expect(r.entries[1].fields["edition"] == "3")166 #expect([r.entries[2].year, r.entries[2].month, r.entries[2].day] == [2020, 6, 1])167 #expect(r.entries[2].fields["number"] == "12")168 #expect(r.entries[3].year == nil)169 #expect(r.entries[3].fields[ReferenceField.issuedLiteral] == "forthcoming")170 #expect(r.entries[4].fields[ReferenceField.rawDate] == "2019-05-01/2019-05-03")171 #expect(r.entries[4].fields["event-date"] == "2019-05-01/2019-05-03")172 #expect(r.entries[5].fields["season"] == "1")173 #expect([r.entries[6].year, r.entries[6].month] == [2001, 2])174 #expect(r.entries[7].key == "roy2005no")175 #expect(r.issues.count == 3) // structured "custom", generated key, non-object item176 #expect(CSLJSON.parse(Data("{ not json".utf8)).issues.count == 1)177 #expect(CSLJSON.parse(Data(#"{"id": "one", "type": "book", "title": "Single object"}"#.utf8)).entries.map(\.key) == ["one"])178 }179180 @Test func roundTrip() throws {181 let first = CSLJSON.parse(Data(Self.sample.utf8)).entries182 let data = try CSLJSON.write(first)183 let second = CSLJSON.parse(data)184 #expect(second.issues.isEmpty, "\(second.issues)")185 #expect(second.entries == first)186 // The output is standard CSL-JSON.187 let json = try #require(JSONSerialization.jsonObject(with: data) as? [[String: Any]])188 let a = json[0]189 #expect((a["issued"] as? [String: Any])?["date-parts"] as? [[Int]] == [[2024, 3, 15]])190 #expect(((a["author"] as? [[String: Any]])?[3])?["suffix"] as? String == "Jr.")191 #expect(((a["author"] as? [[String: Any]])?[4])?["literal"] as? String == "Bank of Canada")192 #expect(((a["translator"] as? [[String: Any]])?[0])?["family"] as? String == "Goldstein")193 #expect((json[4]["issued"] as? [String: Any])?["date-parts"] as? [[Int]] == [[2019, 5, 1], [2019, 5, 3]])194 #expect((json[3]["issued"] as? [String: Any])?["literal"] as? String == "forthcoming")195 }196197 @Test func crossFormatConversionKeepsTheRecord() throws {198 // BibTeX → CSL-JSON → BibTeX and BibTeX → RIS → BibTeX keep what each format can carry.199 let bib = BibTeXParser.parse(BibTeXTests.sample).entries200 let viaJSON = CSLJSON.parse(try CSLJSON.write(bib)).entries201 #expect(viaJSON == bib)202 let viaRIS = RISParser.parse(RISWriter.write(bib)).entries203 #expect(viaRIS.map(\.key) == bib.map(\.key))204 #expect(viaRIS.map(\.type) == bib.map(\.type))205 #expect(viaRIS.map(\.authors) == bib.map(\.authors))206 #expect(viaRIS[0].fields["page"] == "987–1007")207 #expect(viaRIS[0].fields["mycustomfield"] == nil) // documented loss: RIS has no free-form field208 }209}210211@Suite struct ImporterTests {212 @Test func detectsTheFormatFromContentAndExtension() {213 #expect(BibliographyImporter.detectFormat("@article{a, title={T}}", fileExtension: nil) == .bibtex)214 #expect(BibliographyImporter.detectFormat("\u{FEFF} [ {\"id\": 1} ]", fileExtension: "txt") == .cslJSON)215 #expect(BibliographyImporter.detectFormat("Provider: X\nTY - JOUR\nER - ", fileExtension: nil) == .ris)216 #expect(BibliographyImporter.detectFormat("TY - JOUR\nTI - About @article{x} syntax\nER - ", fileExtension: "bib") == .ris) // content wins over a wrong extension217 #expect(BibliographyImporter.detectFormat("", fileExtension: ".BIB") == .bibtex)218 #expect(BibliographyImporter.detectFormat("nothing recognisable", fileExtension: "docx") == nil)219 #expect(BibliographyImporter.importEntries(Data(BibTeXTests.sample.utf8), fileExtension: "bib").entries.count == 4)220 #expect(BibliographyImporter.importEntries(Data(RISTests.sample.utf8), fileExtension: nil).entries.count == 4)221 #expect(BibliographyImporter.importEntries(Data(CSLJSONTests.sample.utf8), fileExtension: "json").entries.count == 8)222 let none = BibliographyImporter.importEntries(Data("hello".utf8), fileExtension: nil)223 #expect(none.entries.isEmpty && none.issues.count == 1)224 // Latin-1 bytes are read, and the user is told.225 let latin = BibliographyImporter.importEntries("@book{a, title = {Été}}".data(using: .isoLatin1)!, fileExtension: "bib")226 #expect(latin.entries.first?.title == "Été")227 #expect(latin.issues.first?.contains("ISO Latin-1") == true)228 }229230 @Test func mergeDeduplicatesAndRenames() {231 let existing = [232 BibEntry(key: "engle1982", type: "article-journal", authors: [person("Engle", "R. F.")], year: 1982, fields: ["title": "Autoregressive Conditional Heteroscedasticity", "DOI": "10.2307/1912773"]),233 BibEntry(key: "smith2020", type: "book", authors: [person("Smith", "A.")], year: 2020, fields: ["title": "The Économie of Things"]),234 ]235 let incoming = [236 BibEntry(key: "engle1982", type: "article-journal", authors: [person("Engle", "Robert")], year: 1982, fields: ["title": "ARCH", "DOI": "https://doi.org/10.2307/1912773"]), // same key, same DOI237 BibEntry(key: "Engle82", type: "article-journal", year: 1982, fields: ["title": "Different title", "DOI": "10.2307/1912773"]), // other key, same DOI238 BibEntry(key: "smith", type: "book", year: 2020, fields: ["title": "the economie of things!"]), // same title + year239 BibEntry(key: "smith2020", type: "book", authors: [person("Smith", "B.")], year: 2020, fields: ["title": "A Completely Different Book"]), // key clash240 BibEntry(key: "smith2020", type: "book", authors: [person("Smith", "C.")], year: 2020, fields: ["title": "A Third Book"]), // key clash again241 BibEntry(key: "new2021", type: "book", year: 2021, fields: ["title": "New"]),242 BibEntry(key: "new2021b", type: "book", year: 2021, fields: ["title": "New"]), // duplicate inside the import243 ]244 let (merged, log) = BibliographyImporter.merge(incoming, into: existing)245 #expect(merged.map(\.key) == ["engle1982", "smith2020", "smith2020-2", "smith2020-3", "new2021"])246 #expect(merged[0] == existing[0] && merged[1] == existing[1]) // existing entries are never modified247 #expect(merged[2].title == "A Completely Different Book")248 #expect(log.count == 7)249 #expect(log[0].contains("same key"))250 #expect(log[1].contains("duplicate of \"engle1982\" (same DOI"))251 #expect(log[2].contains("duplicate of \"smith2020\" (same title and year)"))252 #expect(log[3].contains("imported as \"smith2020-2\""))253 #expect(log[4].contains("imported as \"smith2020-3\""))254 #expect(log[5].contains("duplicate of \"new2021\""))255 #expect(log[6] == "3 added, 4 skipped; the library now has 5 entries.")256 // Deterministic: merging again adds nothing.257 #expect(BibliographyImporter.merge(incoming, into: merged).entries.count == 7 - 2)258 }259260 @Test func missingFieldsAndSuggestedKeys() {261 let complete = BibEntry(key: "k", type: "article-journal", authors: [person("Boucher", "S.-P.")], year: 2024,262 fields: ["title": "On the Volatility of Things", "container-title": "J", "volume": "1", "DOI": "10.1/x"])263 #expect(ReferenceChecks.missingFields(complete).isEmpty)264 #expect(ReferenceChecks.missingFields(BibEntry(key: "k", type: "article-journal")) == ["author", "issued", "title", "container-title", "volume", "page or DOI"])265 #expect(ReferenceChecks.missingFields(BibEntry(key: "k", type: "book", editors: [person("Roe")], year: 2000, fields: ["title": "T"])) == ["publisher"])266 #expect(ReferenceChecks.missingFields(BibEntry(key: "k", type: "chapter", authors: [person("Roe")], year: 2000, fields: ["title": "T", "publisher": "P"])) == ["container-title", "page"])267 #expect(ReferenceChecks.missingFields(BibEntry(key: "k", type: "thesis", authors: [person("Roe")], fields: ["title": "T", ReferenceField.issuedLiteral: "in press"])) == ["genre", "publisher"])268 #expect(ReferenceChecks.missingFields(BibEntry(key: "k", type: "webpage", authors: [org("WHO")], year: 2018, fields: ["title": "T"])) == ["URL"])269270 #expect(ReferenceChecks.suggestedKey(for: complete, existing: []) == "boucher2024volatility")271 #expect(ReferenceChecks.suggestedKey(for: complete, existing: ["boucher2024volatility"]) == "boucher2024volatilitya")272 #expect(ReferenceChecks.suggestedKey(for: complete, existing: ["boucher2024volatility", "boucher2024volatilitya"]) == "boucher2024volatilityb")273 let fr = BibEntry(key: "", type: "book", authors: [person("de Saint-Exupéry", "Antoine")], year: 1943, fields: ["title": "Le Petit Prince"])274 #expect(ReferenceChecks.suggestedKey(for: fr, existing: []) == "saintexupery1943petit")275 #expect(ReferenceChecks.suggestedKey(for: BibEntry(key: "", type: "report", authors: [org("World Health Organization")], fields: ["title": "The top 10"]), existing: []) == "whotop")276 #expect(ReferenceChecks.suggestedKey(for: BibEntry(key: ""), existing: []) == "ref")277 }278}279