SPB Git

Repository Public

InterchangeTests.swift 16.9 KB 279 lines swift

1// ORIX Docs2// File: Packages/ORIXDocsKit/Tests/DocReferencesTests/InterchangeTests.swift3// Description: RIS and CSL-JSON parsing and round trips, format detection, library merging and de-duplication, completeness checks4//              and key suggestion.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import Testing10import DocModel11@testable import DocReferences1213@Suite struct RISTests {14    static let sample = """15    TY  - JOUR16    ID  - bollerslev198617    AU  - Bollerslev, Tim18    A1  - Engle, Robert F., Jr.19    TI  - Generalized autoregressive conditional20          heteroskedasticity21    JO  - Journal of Econometrics22    JA  - J. Econom.23    PY  - 198624    DA  - 1986/04/15/25    VL  - 3126    IS  - 327    SP  - 30728    EP  - 32729    DO  - https://doi.org/10.1016/0304-4076(86)90063-130    UR  - https://example.org/garch31    SN  - 0304-407632    AB  - A natural generalization of the ARCH process.33    KW  - GARCH34    KW  - volatility35    L1  - file:///papers/garch.pdf36    ER  - 3738    TY  - CHAP39    AU  - World Health Organization40    A2  - Roe, Richard41    T1  - A chapter42    T2  - The Book43    PB  - Springer44    CY  - Berlin45    Y1  - 2001///Spring46    SP  - 12-3447    SN  - 978-3-16-148410-048    ET  - 2nd49    ER  -5051    TY  - THES52    AU  - Roy, Anne53    TI  - A thesis54    PB  - Université du Québec en Outaouais55    M3  - PhD thesis56    PY  - in press57    Y2  - 2024/02/03/58    TY  - ZZZZ59    TI  - Unknown type60    """6162    @Test func parsesStandardTags() {63        let r = RISParser.parse(Self.sample)64        #expect(r.entries.count == 4)65        let a = r.entries[0]66        #expect(a.key == "bollerslev1986")67        #expect(a.type == "article-journal")68        #expect(a.authors == [person("Bollerslev", "Tim"), person("Engle", "Robert F., Jr.")])69        #expect(a.title == "Generalized autoregressive conditional heteroskedasticity")          // multi-line value70        #expect(a.fields["container-title"] == "Journal of Econometrics")71        #expect(a.fields["container-title-short"] == "J. Econom.")72        #expect([a.year, a.month, a.day] == [1986, 4, 15])73        #expect(a.fields["volume"] == "31")74        #expect(a.fields["issue"] == "3")75        #expect(a.fields["page"] == "307–327")76        #expect(a.fields["DOI"] == "10.1016/0304-4076(86)90063-1")77        #expect(a.fields["URL"] == "https://example.org/garch")78        #expect(a.fields["ISSN"] == "0304-4076")79        #expect(a.fields["abstract"] == "A natural generalization of the ARCH process.")80        #expect(a.fields["keyword"] == "GARCH; volatility")81        #expect(a.fields["L1"] == "file:///papers/garch.pdf")                                     // unmapped tag kept82        let b = r.entries[1]83        #expect(b.type == "chapter")84        #expect(b.authors == [org("World Health Organization")])85        #expect(b.editors == [person("Roe", "Richard")])86        #expect(b.fields["container-title"] == "The Book")87        #expect(b.fields["publisher"] == "Springer")88        #expect(b.fields["publisher-place"] == "Berlin")89        #expect(b.year == 2001)90        #expect(b.month == nil)91        #expect(b.fields["season"] == "Spring")92        #expect(b.fields["page"] == "12–34")93        #expect(b.fields["ISBN"] == "978-3-16-148410-0")94        #expect(b.fields["edition"] == "2nd")95        #expect(b.key == "who2001chapter")                                                        // generated: the record has no ID96        let c = r.entries[2]97        #expect(c.type == "thesis")98        #expect(c.year == nil)99        #expect(c.fields[ReferenceField.issuedLiteral] == "in press")100        #expect(c.fields["genre"] == "PhD thesis")101        #expect(c.fields["accessed"] == "2024-02-03")102        #expect(r.entries[3].type == "document")103        #expect(r.entries[3].fields["TY"] == "ZZZZ")104        let zLine = Self.sample.split(separator: "\n", omittingEmptySubsequences: false).firstIndex { $0.contains("ZZZZ") }! + 1105        #expect(r.issues.contains { $0.hasPrefix("Line \(zLine):") && $0.contains("before the previous one") })106        #expect(r.issues.contains { $0.contains("unknown reference type \"ZZZZ\"") })107        #expect(r.issues.contains { $0.contains("not closed with ER") })108    }109110    @Test func roundTrip() {111        let first = RISParser.parse(Self.sample).entries112        let written = RISWriter.write(first)113        let second = RISParser.parse(written)114        #expect(second.issues.count == 1 && second.issues[0].contains("ZZZZ"), "\(second.issues)")      // the unknown type is still unknown115        #expect(second.entries == first)116        #expect(written.contains("TY  - JOUR\n"))117        #expect(written.contains("SP  - 307\nEP  - 327\n"))118        #expect(written.contains("TY  - ZZZZ\n"))119        // Type mapping both ways for every CSL type the writer knows.120        for (csl, ris) in RISMapping.fromCSL where !["article", "post", "entry"].contains(csl) {121            #expect(RISMapping.toCSL[ris] == csl, "\(csl) → \(ris)")122            let e = BibEntry(key: "k", type: csl, authors: [person("Roy", "Anne"), person("Plato"), org("Statistics Canada"), .others], year: 2020, fields: ["title": "T"])123            #expect(RISParser.parse(RISWriter.write([e])).entries == [e], "\(csl)")124        }125    }126}127128@Suite struct CSLJSONTests {129    static let sample = #"""130    [131      {"id": "boucher2024", "type": "article-journal", "title": "Volatility <i>in vivo</i>",132       "author": [{"family": "Boucher", "given": "Simon-Pierre"}, {"family": "Gogh", "given": "Vincent", "non-dropping-particle": "van"},133                  {"family": "Beethoven", "given": "Ludwig", "dropping-particle": "van"}, {"family": "King", "given": "Martin Luther", "suffix": "Jr."},134                  {"literal": "Bank of Canada"}],135       "editor": [{"family": "Roe", "given": "R."}],136       "translator": [{"family": "Goldstein", "given": "Ann"}],137       "issued": {"date-parts": [[2024, 3, 15]]}, "accessed": {"date-parts": [["2024", "4", "1"]]},138       "container-title": "Journal of Finance", "volume": 79, "issue": "2", "page": "100-120", "DOI": "https://doi.org/10.1111/jofi.1", "custom": {"a": 1}},139      {"id": 42, "type": "book", "title": "Numbers as strings", "issued": {"date-parts": [["1999"]]}, "edition": 3, "publisher": "P"},140      {"id": "raw", "type": "report", "title": "Raw date", "issued": {"raw": "2020-06-01"}, "number": 12},141      {"id": "lit", "type": "manuscript", "title": "Literal date", "issued": {"literal": "forthcoming"}},142      {"id": "range", "type": "paper-conference", "title": "Range", "issued": {"date-parts": [[2019, 5, 1], [2019, 5, 3]]}, "event-date": {"raw": "2019-05-01/2019-05-03"}},143      {"id": "season", "type": "article-journal", "title": "Season", "issued": {"date-parts": [[2017]], "season": 1}},144      {"id": "str", "type": "book", "title": "String date", "issued": "2001-02"},145      {"type": "book", "title": "No identifier here", "author": [{"family": "Roy", "given": "Anne"}], "issued": {"date-parts": [[2005]]}},146      "not an object"147    ]148    """#149150    @Test func parsesNamesDatesAndLooseTypes() throws {151        let r = CSLJSON.parse(Data(Self.sample.utf8))152        #expect(r.entries.count == 8)153        let a = r.entries[0]154        #expect(a.authors == [person("Boucher", "Simon-Pierre"), person("van Gogh", "Vincent"), person("Beethoven", "Ludwig van"), person("King", "Martin Luther, Jr."),155                              org("Bank of Canada")])156        #expect(a.editors == [person("Roe", "R.")])157        #expect(a.fields["translator"] == "Goldstein, Ann")158        #expect([a.year, a.month, a.day] == [2024, 3, 15])159        #expect(a.fields["accessed"] == "2024-04-01")160        #expect(a.fields["volume"] == "79")161        #expect(a.fields["DOI"] == "10.1111/jofi.1")162        #expect(a.fields["custom"] == #"{"a":1}"#)163        #expect(r.entries[1].key == "42")164        #expect(r.entries[1].year == 1999)165        #expect(r.entries[1].fields["edition"] == "3")166        #expect([r.entries[2].year, r.entries[2].month, r.entries[2].day] == [2020, 6, 1])167        #expect(r.entries[2].fields["number"] == "12")168        #expect(r.entries[3].year == nil)169        #expect(r.entries[3].fields[ReferenceField.issuedLiteral] == "forthcoming")170        #expect(r.entries[4].fields[ReferenceField.rawDate] == "2019-05-01/2019-05-03")171        #expect(r.entries[4].fields["event-date"] == "2019-05-01/2019-05-03")172        #expect(r.entries[5].fields["season"] == "1")173        #expect([r.entries[6].year, r.entries[6].month] == [2001, 2])174        #expect(r.entries[7].key == "roy2005no")175        #expect(r.issues.count == 3)        // structured "custom", generated key, non-object item176        #expect(CSLJSON.parse(Data("{ not json".utf8)).issues.count == 1)177        #expect(CSLJSON.parse(Data(#"{"id": "one", "type": "book", "title": "Single object"}"#.utf8)).entries.map(\.key) == ["one"])178    }179180    @Test func roundTrip() throws {181        let first = CSLJSON.parse(Data(Self.sample.utf8)).entries182        let data = try CSLJSON.write(first)183        let second = CSLJSON.parse(data)184        #expect(second.issues.isEmpty, "\(second.issues)")185        #expect(second.entries == first)186        // The output is standard CSL-JSON.187        let json = try #require(JSONSerialization.jsonObject(with: data) as? [[String: Any]])188        let a = json[0]189        #expect((a["issued"] as? [String: Any])?["date-parts"] as? [[Int]] == [[2024, 3, 15]])190        #expect(((a["author"] as? [[String: Any]])?[3])?["suffix"] as? String == "Jr.")191        #expect(((a["author"] as? [[String: Any]])?[4])?["literal"] as? String == "Bank of Canada")192        #expect(((a["translator"] as? [[String: Any]])?[0])?["family"] as? String == "Goldstein")193        #expect((json[4]["issued"] as? [String: Any])?["date-parts"] as? [[Int]] == [[2019, 5, 1], [2019, 5, 3]])194        #expect((json[3]["issued"] as? [String: Any])?["literal"] as? String == "forthcoming")195    }196197    @Test func crossFormatConversionKeepsTheRecord() throws {198        // BibTeX → CSL-JSON → BibTeX and BibTeX → RIS → BibTeX keep what each format can carry.199        let bib = BibTeXParser.parse(BibTeXTests.sample).entries200        let viaJSON = CSLJSON.parse(try CSLJSON.write(bib)).entries201        #expect(viaJSON == bib)202        let viaRIS = RISParser.parse(RISWriter.write(bib)).entries203        #expect(viaRIS.map(\.key) == bib.map(\.key))204        #expect(viaRIS.map(\.type) == bib.map(\.type))205        #expect(viaRIS.map(\.authors) == bib.map(\.authors))206        #expect(viaRIS[0].fields["page"] == "987–1007")207        #expect(viaRIS[0].fields["mycustomfield"] == nil)        // documented loss: RIS has no free-form field208    }209}210211@Suite struct ImporterTests {212    @Test func detectsTheFormatFromContentAndExtension() {213        #expect(BibliographyImporter.detectFormat("@article{a, title={T}}", fileExtension: nil) == .bibtex)214        #expect(BibliographyImporter.detectFormat("\u{FEFF}  [ {\"id\": 1} ]", fileExtension: "txt") == .cslJSON)215        #expect(BibliographyImporter.detectFormat("Provider: X\nTY  - JOUR\nER  - ", fileExtension: nil) == .ris)216        #expect(BibliographyImporter.detectFormat("TY  - JOUR\nTI  - About @article{x} syntax\nER  - ", fileExtension: "bib") == .ris)   // content wins over a wrong extension217        #expect(BibliographyImporter.detectFormat("", fileExtension: ".BIB") == .bibtex)218        #expect(BibliographyImporter.detectFormat("nothing recognisable", fileExtension: "docx") == nil)219        #expect(BibliographyImporter.importEntries(Data(BibTeXTests.sample.utf8), fileExtension: "bib").entries.count == 4)220        #expect(BibliographyImporter.importEntries(Data(RISTests.sample.utf8), fileExtension: nil).entries.count == 4)221        #expect(BibliographyImporter.importEntries(Data(CSLJSONTests.sample.utf8), fileExtension: "json").entries.count == 8)222        let none = BibliographyImporter.importEntries(Data("hello".utf8), fileExtension: nil)223        #expect(none.entries.isEmpty && none.issues.count == 1)224        // Latin-1 bytes are read, and the user is told.225        let latin = BibliographyImporter.importEntries("@book{a, title = {Été}}".data(using: .isoLatin1)!, fileExtension: "bib")226        #expect(latin.entries.first?.title == "Été")227        #expect(latin.issues.first?.contains("ISO Latin-1") == true)228    }229230    @Test func mergeDeduplicatesAndRenames() {231        let existing = [232            BibEntry(key: "engle1982", type: "article-journal", authors: [person("Engle", "R. F.")], year: 1982, fields: ["title": "Autoregressive Conditional Heteroscedasticity", "DOI": "10.2307/1912773"]),233            BibEntry(key: "smith2020", type: "book", authors: [person("Smith", "A.")], year: 2020, fields: ["title": "The Économie of Things"]),234        ]235        let incoming = [236            BibEntry(key: "engle1982", type: "article-journal", authors: [person("Engle", "Robert")], year: 1982, fields: ["title": "ARCH", "DOI": "https://doi.org/10.2307/1912773"]),   // same key, same DOI237            BibEntry(key: "Engle82", type: "article-journal", year: 1982, fields: ["title": "Different title", "DOI": "10.2307/1912773"]),                                              // other key, same DOI238            BibEntry(key: "smith", type: "book", year: 2020, fields: ["title": "the economie of things!"]),                                                                              // same title + year239            BibEntry(key: "smith2020", type: "book", authors: [person("Smith", "B.")], year: 2020, fields: ["title": "A Completely Different Book"]),                                    // key clash240            BibEntry(key: "smith2020", type: "book", authors: [person("Smith", "C.")], year: 2020, fields: ["title": "A Third Book"]),                                                   // key clash again241            BibEntry(key: "new2021", type: "book", year: 2021, fields: ["title": "New"]),242            BibEntry(key: "new2021b", type: "book", year: 2021, fields: ["title": "New"]),                                                                                               // duplicate inside the import243        ]244        let (merged, log) = BibliographyImporter.merge(incoming, into: existing)245        #expect(merged.map(\.key) == ["engle1982", "smith2020", "smith2020-2", "smith2020-3", "new2021"])246        #expect(merged[0] == existing[0] && merged[1] == existing[1])          // existing entries are never modified247        #expect(merged[2].title == "A Completely Different Book")248        #expect(log.count == 7)249        #expect(log[0].contains("same key"))250        #expect(log[1].contains("duplicate of \"engle1982\" (same DOI"))251        #expect(log[2].contains("duplicate of \"smith2020\" (same title and year)"))252        #expect(log[3].contains("imported as \"smith2020-2\""))253        #expect(log[4].contains("imported as \"smith2020-3\""))254        #expect(log[5].contains("duplicate of \"new2021\""))255        #expect(log[6] == "3 added, 4 skipped; the library now has 5 entries.")256        // Deterministic: merging again adds nothing.257        #expect(BibliographyImporter.merge(incoming, into: merged).entries.count == 7 - 2)258    }259260    @Test func missingFieldsAndSuggestedKeys() {261        let complete = BibEntry(key: "k", type: "article-journal", authors: [person("Boucher", "S.-P.")], year: 2024,262                                fields: ["title": "On the Volatility of Things", "container-title": "J", "volume": "1", "DOI": "10.1/x"])263        #expect(ReferenceChecks.missingFields(complete).isEmpty)264        #expect(ReferenceChecks.missingFields(BibEntry(key: "k", type: "article-journal")) == ["author", "issued", "title", "container-title", "volume", "page or DOI"])265        #expect(ReferenceChecks.missingFields(BibEntry(key: "k", type: "book", editors: [person("Roe")], year: 2000, fields: ["title": "T"])) == ["publisher"])266        #expect(ReferenceChecks.missingFields(BibEntry(key: "k", type: "chapter", authors: [person("Roe")], year: 2000, fields: ["title": "T", "publisher": "P"])) == ["container-title", "page"])267        #expect(ReferenceChecks.missingFields(BibEntry(key: "k", type: "thesis", authors: [person("Roe")], fields: ["title": "T", ReferenceField.issuedLiteral: "in press"])) == ["genre", "publisher"])268        #expect(ReferenceChecks.missingFields(BibEntry(key: "k", type: "webpage", authors: [org("WHO")], year: 2018, fields: ["title": "T"])) == ["URL"])269270        #expect(ReferenceChecks.suggestedKey(for: complete, existing: []) == "boucher2024volatility")271        #expect(ReferenceChecks.suggestedKey(for: complete, existing: ["boucher2024volatility"]) == "boucher2024volatilitya")272        #expect(ReferenceChecks.suggestedKey(for: complete, existing: ["boucher2024volatility", "boucher2024volatilitya"]) == "boucher2024volatilityb")273        let fr = BibEntry(key: "", type: "book", authors: [person("de Saint-Exupéry", "Antoine")], year: 1943, fields: ["title": "Le Petit Prince"])274        #expect(ReferenceChecks.suggestedKey(for: fr, existing: []) == "saintexupery1943petit")275        #expect(ReferenceChecks.suggestedKey(for: BibEntry(key: "", type: "report", authors: [org("World Health Organization")], fields: ["title": "The top 10"]), existing: []) == "whotop")276        #expect(ReferenceChecks.suggestedKey(for: BibEntry(key: ""), existing: []) == "ref")277    }278}279
Files main
Menu

Theme