SPB Git

Repository Public

BibTeXTests.swift 19.2 KB 287 lines swift

1// ORIX Docs2// File: Packages/ORIXDocsKit/Tests/DocReferencesTests/BibTeXTests.swift3// Description: BibTeX / BibLaTeX parser tests on tricky real-world input: macros, concatenation, nested braces, crossref and xdata,4//              LaTeX accents, name parsing, error recovery, and the writer round trip.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import Testing10import DocModel11@testable import DocReferences1213@Suite struct BibTeXTests {14    static let sample = #"""15    % A comment line with an at sign: write to me@example.org16    @preamble{ "\newcommand{\noopsort}[1]{}" }17    @comment{jabref-meta: databaseType:bibtex; {nested {braces}} }18    @string{jfe = "Journal of Financial Economics"}19    @String{ pub = {MIT } # "Press" }2021    @article{engle1982,22      author  = {Engle, Robert F. and Tim Bollerslev and de la Fuente, {\'A}ngel and others},23      title   = "Autoregressive Conditional Heteroscedasticity with Estimates of the Variance of {U}nited {K}ingdom Inflation",24      journal = jfe,25      year    = 1982, volume = 50, number = 4,26      pages   = {987--1007},27      month   = jul,28      doi     = {https://doi.org/10.2307/1912773},29      mycustomfield = {kept {as} is}30    }3132    @Book{knuth1984, author = {Donald E. Knuth}, title = {The {\TeX}book}, publisher = pub, year = {1984}, address = "Reading, MA", edition = {2}}3334    @InProceedings(vaswani2017,35      author = {Vaswani, Ashish and {Google Brain Team}},36      title = {Attention Is All You Need: $O(n^2)$ cost \& more},37      crossref = {nips2017},38      pages = "5998--6008"39    )4041    @proceedings{nips2017,42      title = {Advances in Neural Information Processing Systems 30},43      editor = {Guyon, I. and von Luxburg, U.},44      year = 2017, publisher = {Curran Associates}, address = {Red Hook, NY}45    }46    """#4748    @Test func macrosConcatenationAndDelimiters() {49        let r = BibTeXParser.parse(Self.sample)50        #expect(r.issues.isEmpty, "\(r.issues)")51        #expect(r.entries.map(\.key) == ["engle1982", "knuth1984", "vaswani2017", "nips2017"])52        let engle = r.entries[0]53        #expect(engle.type == "article-journal")54        #expect(engle.fields["container-title"] == "Journal of Financial Economics")55        #expect(engle.title == "Autoregressive Conditional Heteroscedasticity with Estimates of the Variance of United Kingdom Inflation")56        #expect(engle.year == 1982)57        #expect(engle.month == 7)58        #expect(engle.fields["volume"] == "50")59        #expect(engle.fields["issue"] == "4")60        #expect(engle.fields["page"] == "987–1007")61        #expect(engle.fields["DOI"] == "10.2307/1912773")62        #expect(engle.fields["mycustomfield"] == "kept as is")63        #expect(engle.authors == [person("Engle", "Robert F."), person("Bollerslev", "Tim"), person("de la Fuente", "Ángel"), .others])64        let knuth = r.entries[1]65        #expect(knuth.type == "book")66        #expect(knuth.authors == [person("Knuth", "Donald E.")])67        #expect(knuth.title == "The TeXbook")68        #expect(knuth.fields["publisher"] == "MIT Press")69        #expect(knuth.fields["publisher-place"] == "Reading, MA")70        #expect(knuth.fields["edition"] == "2")71    }7273    @Test func crossrefInheritsFromTheParent() {74        let r = BibTeXParser.parse(Self.sample)75        let v = r.entries[2]76        #expect(v.type == "paper-conference")77        #expect(v.authors == [person("Vaswani", "Ashish"), org("Google Brain Team")])78        #expect(v.title == "Attention Is All You Need: $O(n^2)$ cost & more")          // math untouched, \& decoded79        #expect(v.fields["container-title"] == "Advances in Neural Information Processing Systems 30")80        #expect(v.year == 2017)81        #expect(v.fields["publisher"] == "Curran Associates")82        #expect(v.editors == [person("Guyon", "I."), person("von Luxburg", "U.")])83        #expect(v.fields["crossref"] == "nips2017")84        #expect(v.fields["page"] == "5998–6008")85        // The parent keeps its own title.86        #expect(r.entries[3].title == "Advances in Neural Information Processing Systems 30")87        #expect(r.entries[3].fields["container-title"] == nil)88    }8990    @Test func xdataAndMissingParents() {91        let r = BibTeXParser.parse(#"""92        @xdata{press, publisher = {Presses de l'Universit{\'e} Laval}, location = {Qu{\'e}bec}}93        @book{a, title = {Un}, xdata = {press}, year = 2001}94        @book{b, title = {Deux}, xdata = {press, nowhere}, publisher = {Autre}, year = 2002}95        @inbook{c, title = {Trois}, crossref = {ghost}}96        """#)97        #expect(r.entries.map(\.key) == ["a", "b", "c"])98        #expect(r.entries[0].fields["publisher"] == "Presses de l'Université Laval")99        #expect(r.entries[0].fields["publisher-place"] == "Québec")100        #expect(r.entries[0].fields["xdata"] == nil)101        #expect(r.entries[1].fields["publisher"] == "Autre")102        #expect(r.entries[1].fields["publisher-place"] == "Québec")103        #expect(r.issues.count == 2)104        #expect(r.issues.contains { $0.contains("Line 3") && $0.contains("xdata \"nowhere\"") })105        #expect(r.issues.contains { $0.contains("Line 4") && $0.contains("crossref \"ghost\"") })106    }107108    @Test func latexDecoding() {109        let cases: [(String, String)] = [110            (#"{\'e}l{\`e}ve"#, "élève"), (#"\"{o}"#, "ö"), (#"G{\"o}del"#, "Gödel"), (#"Stra\ss e"#, "Straße"), (#"gar\c{c}on"#, "garçon"),111            (#"\v{S}koda"#, "Škoda"), (#"\v Z"#, "Ž"), (#"Espa\~na"#, "España"), (#"na\"\i ve"#, "naïve"), (#"{\aa}ngstr{\"o}m"#, "ångström"),112            (#"S{\o}ren"#, "Søren"), (#"\L{}\'od\'z"#, "Łódź"), (#"Erd\H{o}s"#, "Erdős"), (#"\^{\i}le"#, "île"), (#"\AE sop"#, "Æsop"),113            ("pages 12--34", "pages 12–34"), ("a---b", "a—b"), ("J.~K. Rowling", "J.\u{00A0}K. Rowling"), (#"Tom \& Jerry"#, "Tom & Jerry"),114            (#"100\% of \$5"#, "100% of $5"), ("``quoted''", "“quoted”"), (#"\emph{Homo sapiens} in {E}urope"#, "<i>Homo sapiens</i> in Europe"),115            (#"{\em Homo} {\bf bold}"#, "<i>Homo</i> <b>bold</b>"), (#"H\textsubscript{2}O"#, "H<sub>2</sub>O"),116            (#"the $\alpha$-stable {L}\'evy process"#, #"the $\alpha$-stable Lévy process"#),          // math left untouched117            (#"energy $E = mc^{2}$ \(x_{i}\)"#, #"energy $E = mc^{2}$ \(x_{i}\)"#),118            (#"\unknowncmd{arg} stays"#, #"\unknowncmd{arg} stays"#),                                   // unknown commands are not guessed119            ("  several\n   lines  ", "several lines"), (#"\textsc{nasa}"#, LaTeXText.smallCapsOpen + "nasa" + LaTeXText.smallCapsClose),120            (#"\url{http://a.b/c_d}"#, "http://a.b/c_d"), (#"\href{http://a.b}{text}"#, "text"), (#"caf\'e"#, "café"), (#"\`A la carte"#, "À la carte"),121        ]122        for (raw, expected) in cases { #expect(LaTeXText.decode(raw) == expected, "\(raw)") }123    }124125    @Test func latexEncodeDecodeRoundTrip() {126        let fixed = ["Tom & Jerry: 100% #1_x", "a -- b – c — d --- e", "“x” ‘y’ `` '' ' `", "$x^2$ and $5 or $6, US$5–$10", "back\\slash \\ss \\foo{bar} \\textit",127                     "{braces} } {", "<i>Homo sapiens</i> & <b>bold</b> <sup>2</sup>", "tilde ~ and\u{00A0}nbsp", "Gödel, Erdős, Škoda, naïve, Łódź", "a\u{2009}b\u{00AD}c",128                     "\\(x\\) and \\[y\\]", "-–-—--", "<i>unbalanced", "a < b > c"]129        for s in fixed { #expect(LaTeXText.decode(LaTeXText.encode(s)) == s, "\(s) → \(LaTeXText.encode(s))") }130        // Random strings over an alphabet of characters that matter to BibTeX.131        var rng = SeededGenerator(seed: 20261121)132        let alphabet: [String] = ["a", "B", " ", "{", "}", "\\", "$", "%", "&", "#", "_", "~", "-", "–", "—", "`", "'", "“", "”", "é", "<i>", "</i>", "^", "1", "ss", "\"", ","]133        var failures: [String] = []134        for _ in 0..<20000 {135            let s = LaTeXText.collapse((0..<Int.random(in: 1...14, using: &rng)).map { _ in alphabet.randomElement(using: &rng)! }.joined())136            let encoded = BibTeXWriter.encodeValue(s)137            if LaTeXText.decode(encoded) != s { failures.append("\(s) → \(encoded) → \(LaTeXText.decode(encoded))") }138        }139        #expect(failures.isEmpty, "\(failures.prefix(5))")140    }141142    @Test func nameParsing() {143        func names(_ s: String) -> [PersonName] { NameParser.parseList(s, decode: { LaTeXText.decode($0) }) }144        #expect(names("Ludwig van Beethoven") == [person("van Beethoven", "Ludwig")])145        #expect(names("van Beethoven, Ludwig") == [person("van Beethoven", "Ludwig")])146        #expect(names("Jean de La Fontaine") == [person("de La Fontaine", "Jean")])147        #expect(names("King, Jr., Martin Luther") == [person("King", "Martin Luther, Jr.")])148        #expect(names("Charles Louis Xavier Joseph de la Vall{\\'e}e Poussin") == [person("de la Vallée Poussin", "Charles Louis Xavier Joseph")])149        #expect(names("{World Health Organization}") == [org("World Health Organization")])150        #expect(names("{Barnes and Noble, Inc.} and Smith, John and others") == [org("Barnes and Noble, Inc."), person("Smith", "John"), .others])151        #expect(names("Brinch Hansen, Per") == [person("Brinch Hansen", "Per")])152        #expect(names("Per {Brinch Hansen}") == [person("Brinch Hansen", "Per")])153        #expect(names("Aristotle") == [person("Aristotle")])154        #expect(names("J.~K. Rowling AND Tolkien, J. R. R.") == [person("Rowling", "J. K."), person("Tolkien", "J. R. R.")])155        #expect(names("{\\'E}mile Zola\n   and Sand, George") == [person("Zola", "Émile"), person("Sand", "George")])156        #expect(names("Anderson") == [person("Anderson")])     // "and" inside a word does not split157        #expect(PersonName(family: "King", given: "Martin Luther, Jr.").givenAndSuffix == ("Martin Luther", "Jr."))158        // Serialisation is the inverse.159        let list = [person("van Beethoven", "Ludwig"), person("King", "Martin Luther, Jr."), org("World Health Organization"), person("De La Cruz"), person("Aristotle"), .others]160        #expect(NameParser.parseList(NameParser.string(list)) == list)161    }162163    @Test func bibLaTeXTypesFieldsAndDates() {164        let r = BibTeXParser.parse(#"""165        @online{who2018, author = {{World Health Organization}}, title = {The top 10 causes of death}, date = {2018-05-24},166                url = {https://www.who.int/news-room/fact-sheets/detail/the-top-10-causes-of-death}, urldate = {2020-01-09}}167        @thesis{harris2014, author = {Harris, L.}, title = {Instructional leadership}, subtitle = {Perceptions and practices}, type = {phdthesis},168                institution = {University of Virginia}, location = {Charlottesville}, date = {2014}}169        @mastersthesis{m, author = {Roy, A.}, title = {T}, school = {UQO}, year = 2020}170        @inproceedings{conf, author = {Doe, Jane}, title = {A Talk}, booktitle = {Proceedings of X}, eventtitle = {X 2019}, eventdate = {2019-05-01/2019-05-03},171                venue = {Montréal}, date = {2019-05-01/2019-05-03}, pages = {1-9}}172        @article{mag, author = {Doe, Jane}, title = {News}, journaltitle = {The Magazine}, journalsubtitle = {A Weekly}, entrysubtype = {magazine}, date = {2021-03},173                 number = {12}, issue = {Spring}, eid = {e42}}174        @report{rep, author = {Doe, Jane}, title = {R}, type = {techreport}, institution = {Bank of Canada}, number = {2020-5}, year = {2020}}175        @incollection{chap, author = {Doe, Jane}, title = {C}, booktitle = {B}, editor = {Roe, R.}, publisher = {P}, year = {forthcoming}, series = {Lecture Notes}, number = {7}}176        @misc{arx, author = {Doe, Jane}, title = {Preprint}, year = 2023, eprint = {2301.00001}, eprinttype = {arXiv}, howpublished = {\url{https://arxiv.org/abs/2301.00001}}}177        @artwork{art, author = {Doe, Jane}, title = {Painting}, date = {1889-06~}}178        @unknowntype{u, title = {Mystery}, year = 1999}179        """#)180        let e = Dictionary(uniqueKeysWithValues: r.entries.map { ($0.key, $0) })181        #expect(e["who2018"]?.type == "webpage")182        #expect(e["who2018"]?.authors == [org("World Health Organization")])183        #expect([e["who2018"]?.year, e["who2018"]?.month, e["who2018"]?.day] == [2018, 5, 24])184        #expect(e["who2018"]?.fields["accessed"] == "2020-01-09")185        #expect(e["who2018"]?.fields[ReferenceField.rawDate] == nil)186        #expect(e["harris2014"]?.type == "thesis")187        #expect(e["harris2014"]?.title == "Instructional leadership: Perceptions and practices")188        #expect(e["harris2014"]?.fields["genre"] == "PhD thesis")189        #expect(e["harris2014"]?.fields["publisher"] == "University of Virginia")190        #expect(e["harris2014"]?.fields["publisher-place"] == "Charlottesville")191        #expect(e["m"]?.fields["genre"] == "Master's thesis")192        #expect(e["m"]?.fields["publisher"] == "UQO")193        #expect(e["conf"]?.type == "paper-conference")194        #expect([e["conf"]?.year, e["conf"]?.month, e["conf"]?.day] == [2019, 5, 1])195        #expect(e["conf"]?.fields[ReferenceField.rawDate] == "2019-05-01/2019-05-03")     // the range is not lost196        #expect(e["conf"]?.fields["event-title"] == "X 2019")197        #expect(e["conf"]?.fields["event-place"] == "Montréal")198        #expect(e["conf"]?.fields["container-title"] == "Proceedings of X")199        #expect(e["mag"]?.type == "article-magazine")200        #expect(e["mag"]?.fields["container-title"] == "The Magazine: A Weekly")201        #expect(e["mag"]?.fields["number"] == "12")202        #expect(e["mag"]?.fields["issue"] == "Spring")203        #expect(e["mag"]?.fields[ReferenceField.articleNumber] == "e42")204        #expect([e["mag"]?.year, e["mag"]?.month, e["mag"]?.day] == [2021, 3, nil])205        #expect(e["rep"]?.type == "report")206        #expect(e["rep"]?.fields["genre"] == "Technical report")207        #expect(e["rep"]?.fields["publisher"] == "Bank of Canada")208        #expect(e["rep"]?.fields["number"] == "2020-5")209        #expect(e["chap"]?.type == "chapter")210        #expect(e["chap"]?.year == nil)211        #expect(e["chap"]?.fields[ReferenceField.issuedLiteral] == "forthcoming")       // not turned into a number212        #expect(e["chap"]?.fields["collection-title"] == "Lecture Notes")213        #expect(e["chap"]?.fields["collection-number"] == "7")214        #expect(e["arx"]?.type == "document")215        #expect(e["arx"]?.fields["URL"] == "https://arxiv.org/abs/2301.00001")216        #expect(e["arx"]?.fields["eprint"] == "2301.00001")217        #expect(e["arx"]?.fields["eprinttype"] == "arXiv")218        #expect(e["art"]?.type == "graphic")219        #expect([e["art"]?.year, e["art"]?.month] == [1889, 6])220        #expect(e["art"]?.fields[ReferenceField.rawDate] == "1889-06~")                 // approximate date kept221        #expect(e["u"]?.type == "document")222        #expect(e["u"]?.fields[ReferenceField.entryType] == "unknowntype")223        #expect(r.issues.count == 1)224        #expect(r.issues[0].contains("Line 14") && r.issues[0].contains("@unknowntype"))225    }226227    @Test func errorsAreReportedWithLineNumbersAndParsingResumes() {228        let r = BibTeXParser.parse(#"""229        @article{good1, title = {One}, year = 2001}230        @article{broken, title = {Two} year = 2002}231        @article{good2, title = {Three}, year = {2003}, month = sept, month = oct, note = undefinedmacro}232        @book{good1, title = {Duplicate}}233        @book{nokey2, title = {Unclosed {brace}, year = 2004}234        @book{good3, title = {Four}}235        """#)236        #expect(r.entries.map(\.key) == ["good1", "broken", "good2", "nokey2", "good3"])237        #expect(r.entries[3].title == "Unclosed brace, year = 2004")238        #expect(r.entries[1].title == "Two")                    // what was read before the error is kept239        #expect(r.entries[1].year == nil)240        #expect(r.entries[2].month == 9)241        #expect(r.entries[2].fields["note"] == "undefinedmacro")242        #expect(r.issues.contains { $0.hasPrefix("Line 2:") && $0.contains("missing comma after field \"title\"") })243        #expect(r.issues.contains { $0.hasPrefix("Line 3:") && $0.contains("\"month\" is given twice") })244        #expect(r.issues.contains { $0.hasPrefix("Line 3:") && $0.contains("undefined macro \"undefinedmacro\"") })245        #expect(r.issues.contains { $0.hasPrefix("Line 4:") && $0.contains("duplicate key \"good1\"") })246        #expect(r.issues.contains { $0.hasPrefix("Line 6:") && $0.contains("entry \"nokey2\": missing comma") })247        let open = BibTeXParser.parse("@book{a, title = {Fine}}\n@book{x,\n  title = {Never closed")248        #expect(open.entries.map(\.key) == ["a"])249        #expect(open.issues.count == 1 && open.issues[0].hasPrefix("Line 3:") && open.issues[0].contains("unbalanced braces"))250        // Nothing is made up for an empty or hopeless input.251        #expect(BibTeXParser.parse("").entries.isEmpty)252        #expect(BibTeXParser.parse("just text, no entries @ all").entries.isEmpty)253        let noKey = BibTeXParser.parse("@book{, author = {Roy, Anne}, title = {The Long Winter}, year = 1999}")254        #expect(noKey.entries.map(\.key) == ["roy1999long"])255        #expect(noKey.issues.count == 1)256    }257258    @Test func writerRoundTrip() {259        let first = BibTeXParser.parse(Self.sample).entries260        let written = BibTeXWriter.write(first)261        let second = BibTeXParser.parse(written)262        #expect(second.issues.isEmpty, "\(second.issues)")263        #expect(second.entries == first)264        #expect(written.contains("mycustomfield = {kept as is}"))265        #expect(written.contains("month = jul"))266        #expect(written.contains("author = {Engle, Robert F. and Bollerslev, Tim and de la Fuente, Ángel and others}"))267        #expect(written.contains("{World Health Organization}") == false)268        #expect(written.contains("author = {Vaswani, Ashish and {Google Brain Team}}"))269        // Entries built in code, with awkward content, survive too.270        let odd = [271            BibEntry(key: "odd1", type: "article-magazine", authors: [org("Barnes and Noble, Inc."), person("O'Neil", "Shaquille, Jr."), .others],272                     editors: [person("De La Cruz")], year: 2020, month: 12, day: 31,273                     fields: ["title": "NASA & the iPhone: 100% {weird} $x_1$ — “quoted” <i>Homo sapiens</i>", "container-title": "Wired", "issue": "5", "number": "N-12",274                              "page": "12–14, 20", "URL": "https://example.org/a_b?c=d%20e#f", "DOI": "10.1000/xyz_(123)", "abstract": "Line one.\nLine two.",275                              "translator": "Goldstein, Ann and {Some Collective}", "accessed": "2021-01-02", "x-custom": "value", "keyword": "a, b; c"]),276            BibEntry(key: "odd2", type: "speech", fields: ["title": "A speech", ReferenceField.issuedLiteral: "in press"]),277            BibEntry(key: "odd3", type: "thesis", authors: [person("Roy", "A.")], year: 2020, fields: ["title": "T", "genre": "Doctoral dissertation", "publisher": "UQO"]),278            BibEntry(key: "odd4", type: "report", year: 1850, fields: ["title": "R", ReferenceField.rawDate: "1850~", "collection-title": "Series", "number": "4"]),279        ]280        let again = BibTeXParser.parse(BibTeXWriter.write(odd))281        #expect(again.issues.isEmpty, "\(again.issues)")282        #expect(again.entries == odd)283        #expect(BibTeXWriter.write(odd).contains("title = {{NASA} \\& the {iPhone}:") == false)   // titles with markup or math are not case-protected284        #expect(BibTeXWriter.write([BibEntry(key: "k", type: "book", fields: ["title": "GARCH models at NASA"])]).contains("title = {{GARCH} models at {NASA}}"))285    }286}287
Files main
Menu

Theme