Repository Public
1// ORIX Docs2// File: Packages/ORIXDocsKit/Tests/DocReferencesTests/BibTeXTests.swift3// Description: BibTeX / BibLaTeX parser tests on tricky real-world input: macros, concatenation, nested braces, crossref and xdata,4// LaTeX accents, name parsing, error recovery, and the writer round trip.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import Testing10import DocModel11@testable import DocReferences1213@Suite struct BibTeXTests {14 static let sample = #"""15 % A comment line with an at sign: write to me@example.org16 @preamble{ "\newcommand{\noopsort}[1]{}" }17 @comment{jabref-meta: databaseType:bibtex; {nested {braces}} }18 @string{jfe = "Journal of Financial Economics"}19 @String{ pub = {MIT } # "Press" }2021 @article{engle1982,22 author = {Engle, Robert F. and Tim Bollerslev and de la Fuente, {\'A}ngel and others},23 title = "Autoregressive Conditional Heteroscedasticity with Estimates of the Variance of {U}nited {K}ingdom Inflation",24 journal = jfe,25 year = 1982, volume = 50, number = 4,26 pages = {987--1007},27 month = jul,28 doi = {https://doi.org/10.2307/1912773},29 mycustomfield = {kept {as} is}30 }3132 @Book{knuth1984, author = {Donald E. Knuth}, title = {The {\TeX}book}, publisher = pub, year = {1984}, address = "Reading, MA", edition = {2}}3334 @InProceedings(vaswani2017,35 author = {Vaswani, Ashish and {Google Brain Team}},36 title = {Attention Is All You Need: $O(n^2)$ cost \& more},37 crossref = {nips2017},38 pages = "5998--6008"39 )4041 @proceedings{nips2017,42 title = {Advances in Neural Information Processing Systems 30},43 editor = {Guyon, I. and von Luxburg, U.},44 year = 2017, publisher = {Curran Associates}, address = {Red Hook, NY}45 }46 """#4748 @Test func macrosConcatenationAndDelimiters() {49 let r = BibTeXParser.parse(Self.sample)50 #expect(r.issues.isEmpty, "\(r.issues)")51 #expect(r.entries.map(\.key) == ["engle1982", "knuth1984", "vaswani2017", "nips2017"])52 let engle = r.entries[0]53 #expect(engle.type == "article-journal")54 #expect(engle.fields["container-title"] == "Journal of Financial Economics")55 #expect(engle.title == "Autoregressive Conditional Heteroscedasticity with Estimates of the Variance of United Kingdom Inflation")56 #expect(engle.year == 1982)57 #expect(engle.month == 7)58 #expect(engle.fields["volume"] == "50")59 #expect(engle.fields["issue"] == "4")60 #expect(engle.fields["page"] == "987–1007")61 #expect(engle.fields["DOI"] == "10.2307/1912773")62 #expect(engle.fields["mycustomfield"] == "kept as is")63 #expect(engle.authors == [person("Engle", "Robert F."), person("Bollerslev", "Tim"), person("de la Fuente", "Ángel"), .others])64 let knuth = r.entries[1]65 #expect(knuth.type == "book")66 #expect(knuth.authors == [person("Knuth", "Donald E.")])67 #expect(knuth.title == "The TeXbook")68 #expect(knuth.fields["publisher"] == "MIT Press")69 #expect(knuth.fields["publisher-place"] == "Reading, MA")70 #expect(knuth.fields["edition"] == "2")71 }7273 @Test func crossrefInheritsFromTheParent() {74 let r = BibTeXParser.parse(Self.sample)75 let v = r.entries[2]76 #expect(v.type == "paper-conference")77 #expect(v.authors == [person("Vaswani", "Ashish"), org("Google Brain Team")])78 #expect(v.title == "Attention Is All You Need: $O(n^2)$ cost & more") // math untouched, \& decoded79 #expect(v.fields["container-title"] == "Advances in Neural Information Processing Systems 30")80 #expect(v.year == 2017)81 #expect(v.fields["publisher"] == "Curran Associates")82 #expect(v.editors == [person("Guyon", "I."), person("von Luxburg", "U.")])83 #expect(v.fields["crossref"] == "nips2017")84 #expect(v.fields["page"] == "5998–6008")85 // The parent keeps its own title.86 #expect(r.entries[3].title == "Advances in Neural Information Processing Systems 30")87 #expect(r.entries[3].fields["container-title"] == nil)88 }8990 @Test func xdataAndMissingParents() {91 let r = BibTeXParser.parse(#"""92 @xdata{press, publisher = {Presses de l'Universit{\'e} Laval}, location = {Qu{\'e}bec}}93 @book{a, title = {Un}, xdata = {press}, year = 2001}94 @book{b, title = {Deux}, xdata = {press, nowhere}, publisher = {Autre}, year = 2002}95 @inbook{c, title = {Trois}, crossref = {ghost}}96 """#)97 #expect(r.entries.map(\.key) == ["a", "b", "c"])98 #expect(r.entries[0].fields["publisher"] == "Presses de l'Université Laval")99 #expect(r.entries[0].fields["publisher-place"] == "Québec")100 #expect(r.entries[0].fields["xdata"] == nil)101 #expect(r.entries[1].fields["publisher"] == "Autre")102 #expect(r.entries[1].fields["publisher-place"] == "Québec")103 #expect(r.issues.count == 2)104 #expect(r.issues.contains { $0.contains("Line 3") && $0.contains("xdata \"nowhere\"") })105 #expect(r.issues.contains { $0.contains("Line 4") && $0.contains("crossref \"ghost\"") })106 }107108 @Test func latexDecoding() {109 let cases: [(String, String)] = [110 (#"{\'e}l{\`e}ve"#, "élève"), (#"\"{o}"#, "ö"), (#"G{\"o}del"#, "Gödel"), (#"Stra\ss e"#, "Straße"), (#"gar\c{c}on"#, "garçon"),111 (#"\v{S}koda"#, "Škoda"), (#"\v Z"#, "Ž"), (#"Espa\~na"#, "España"), (#"na\"\i ve"#, "naïve"), (#"{\aa}ngstr{\"o}m"#, "ångström"),112 (#"S{\o}ren"#, "Søren"), (#"\L{}\'od\'z"#, "Łódź"), (#"Erd\H{o}s"#, "Erdős"), (#"\^{\i}le"#, "île"), (#"\AE sop"#, "Æsop"),113 ("pages 12--34", "pages 12–34"), ("a---b", "a—b"), ("J.~K. Rowling", "J.\u{00A0}K. Rowling"), (#"Tom \& Jerry"#, "Tom & Jerry"),114 (#"100\% of \$5"#, "100% of $5"), ("``quoted''", "“quoted”"), (#"\emph{Homo sapiens} in {E}urope"#, "<i>Homo sapiens</i> in Europe"),115 (#"{\em Homo} {\bf bold}"#, "<i>Homo</i> <b>bold</b>"), (#"H\textsubscript{2}O"#, "H<sub>2</sub>O"),116 (#"the $\alpha$-stable {L}\'evy process"#, #"the $\alpha$-stable Lévy process"#), // math left untouched117 (#"energy $E = mc^{2}$ \(x_{i}\)"#, #"energy $E = mc^{2}$ \(x_{i}\)"#),118 (#"\unknowncmd{arg} stays"#, #"\unknowncmd{arg} stays"#), // unknown commands are not guessed119 (" several\n lines ", "several lines"), (#"\textsc{nasa}"#, LaTeXText.smallCapsOpen + "nasa" + LaTeXText.smallCapsClose),120 (#"\url{http://a.b/c_d}"#, "http://a.b/c_d"), (#"\href{http://a.b}{text}"#, "text"), (#"caf\'e"#, "café"), (#"\`A la carte"#, "À la carte"),121 ]122 for (raw, expected) in cases { #expect(LaTeXText.decode(raw) == expected, "\(raw)") }123 }124125 @Test func latexEncodeDecodeRoundTrip() {126 let fixed = ["Tom & Jerry: 100% #1_x", "a -- b – c — d --- e", "“x” ‘y’ `` '' ' `", "$x^2$ and $5 or $6, US$5–$10", "back\\slash \\ss \\foo{bar} \\textit",127 "{braces} } {", "<i>Homo sapiens</i> & <b>bold</b> <sup>2</sup>", "tilde ~ and\u{00A0}nbsp", "Gödel, Erdős, Škoda, naïve, Łódź", "a\u{2009}b\u{00AD}c",128 "\\(x\\) and \\[y\\]", "-–-—--", "<i>unbalanced", "a < b > c"]129 for s in fixed { #expect(LaTeXText.decode(LaTeXText.encode(s)) == s, "\(s) → \(LaTeXText.encode(s))") }130 // Random strings over an alphabet of characters that matter to BibTeX.131 var rng = SeededGenerator(seed: 20261121)132 let alphabet: [String] = ["a", "B", " ", "{", "}", "\\", "$", "%", "&", "#", "_", "~", "-", "–", "—", "`", "'", "“", "”", "é", "<i>", "</i>", "^", "1", "ss", "\"", ","]133 var failures: [String] = []134 for _ in 0..<20000 {135 let s = LaTeXText.collapse((0..<Int.random(in: 1...14, using: &rng)).map { _ in alphabet.randomElement(using: &rng)! }.joined())136 let encoded = BibTeXWriter.encodeValue(s)137 if LaTeXText.decode(encoded) != s { failures.append("\(s) → \(encoded) → \(LaTeXText.decode(encoded))") }138 }139 #expect(failures.isEmpty, "\(failures.prefix(5))")140 }141142 @Test func nameParsing() {143 func names(_ s: String) -> [PersonName] { NameParser.parseList(s, decode: { LaTeXText.decode($0) }) }144 #expect(names("Ludwig van Beethoven") == [person("van Beethoven", "Ludwig")])145 #expect(names("van Beethoven, Ludwig") == [person("van Beethoven", "Ludwig")])146 #expect(names("Jean de La Fontaine") == [person("de La Fontaine", "Jean")])147 #expect(names("King, Jr., Martin Luther") == [person("King", "Martin Luther, Jr.")])148 #expect(names("Charles Louis Xavier Joseph de la Vall{\\'e}e Poussin") == [person("de la Vallée Poussin", "Charles Louis Xavier Joseph")])149 #expect(names("{World Health Organization}") == [org("World Health Organization")])150 #expect(names("{Barnes and Noble, Inc.} and Smith, John and others") == [org("Barnes and Noble, Inc."), person("Smith", "John"), .others])151 #expect(names("Brinch Hansen, Per") == [person("Brinch Hansen", "Per")])152 #expect(names("Per {Brinch Hansen}") == [person("Brinch Hansen", "Per")])153 #expect(names("Aristotle") == [person("Aristotle")])154 #expect(names("J.~K. Rowling AND Tolkien, J. R. R.") == [person("Rowling", "J. K."), person("Tolkien", "J. R. R.")])155 #expect(names("{\\'E}mile Zola\n and Sand, George") == [person("Zola", "Émile"), person("Sand", "George")])156 #expect(names("Anderson") == [person("Anderson")]) // "and" inside a word does not split157 #expect(PersonName(family: "King", given: "Martin Luther, Jr.").givenAndSuffix == ("Martin Luther", "Jr."))158 // Serialisation is the inverse.159 let list = [person("van Beethoven", "Ludwig"), person("King", "Martin Luther, Jr."), org("World Health Organization"), person("De La Cruz"), person("Aristotle"), .others]160 #expect(NameParser.parseList(NameParser.string(list)) == list)161 }162163 @Test func bibLaTeXTypesFieldsAndDates() {164 let r = BibTeXParser.parse(#"""165 @online{who2018, author = {{World Health Organization}}, title = {The top 10 causes of death}, date = {2018-05-24},166 url = {https://www.who.int/news-room/fact-sheets/detail/the-top-10-causes-of-death}, urldate = {2020-01-09}}167 @thesis{harris2014, author = {Harris, L.}, title = {Instructional leadership}, subtitle = {Perceptions and practices}, type = {phdthesis},168 institution = {University of Virginia}, location = {Charlottesville}, date = {2014}}169 @mastersthesis{m, author = {Roy, A.}, title = {T}, school = {UQO}, year = 2020}170 @inproceedings{conf, author = {Doe, Jane}, title = {A Talk}, booktitle = {Proceedings of X}, eventtitle = {X 2019}, eventdate = {2019-05-01/2019-05-03},171 venue = {Montréal}, date = {2019-05-01/2019-05-03}, pages = {1-9}}172 @article{mag, author = {Doe, Jane}, title = {News}, journaltitle = {The Magazine}, journalsubtitle = {A Weekly}, entrysubtype = {magazine}, date = {2021-03},173 number = {12}, issue = {Spring}, eid = {e42}}174 @report{rep, author = {Doe, Jane}, title = {R}, type = {techreport}, institution = {Bank of Canada}, number = {2020-5}, year = {2020}}175 @incollection{chap, author = {Doe, Jane}, title = {C}, booktitle = {B}, editor = {Roe, R.}, publisher = {P}, year = {forthcoming}, series = {Lecture Notes}, number = {7}}176 @misc{arx, author = {Doe, Jane}, title = {Preprint}, year = 2023, eprint = {2301.00001}, eprinttype = {arXiv}, howpublished = {\url{https://arxiv.org/abs/2301.00001}}}177 @artwork{art, author = {Doe, Jane}, title = {Painting}, date = {1889-06~}}178 @unknowntype{u, title = {Mystery}, year = 1999}179 """#)180 let e = Dictionary(uniqueKeysWithValues: r.entries.map { ($0.key, $0) })181 #expect(e["who2018"]?.type == "webpage")182 #expect(e["who2018"]?.authors == [org("World Health Organization")])183 #expect([e["who2018"]?.year, e["who2018"]?.month, e["who2018"]?.day] == [2018, 5, 24])184 #expect(e["who2018"]?.fields["accessed"] == "2020-01-09")185 #expect(e["who2018"]?.fields[ReferenceField.rawDate] == nil)186 #expect(e["harris2014"]?.type == "thesis")187 #expect(e["harris2014"]?.title == "Instructional leadership: Perceptions and practices")188 #expect(e["harris2014"]?.fields["genre"] == "PhD thesis")189 #expect(e["harris2014"]?.fields["publisher"] == "University of Virginia")190 #expect(e["harris2014"]?.fields["publisher-place"] == "Charlottesville")191 #expect(e["m"]?.fields["genre"] == "Master's thesis")192 #expect(e["m"]?.fields["publisher"] == "UQO")193 #expect(e["conf"]?.type == "paper-conference")194 #expect([e["conf"]?.year, e["conf"]?.month, e["conf"]?.day] == [2019, 5, 1])195 #expect(e["conf"]?.fields[ReferenceField.rawDate] == "2019-05-01/2019-05-03") // the range is not lost196 #expect(e["conf"]?.fields["event-title"] == "X 2019")197 #expect(e["conf"]?.fields["event-place"] == "Montréal")198 #expect(e["conf"]?.fields["container-title"] == "Proceedings of X")199 #expect(e["mag"]?.type == "article-magazine")200 #expect(e["mag"]?.fields["container-title"] == "The Magazine: A Weekly")201 #expect(e["mag"]?.fields["number"] == "12")202 #expect(e["mag"]?.fields["issue"] == "Spring")203 #expect(e["mag"]?.fields[ReferenceField.articleNumber] == "e42")204 #expect([e["mag"]?.year, e["mag"]?.month, e["mag"]?.day] == [2021, 3, nil])205 #expect(e["rep"]?.type == "report")206 #expect(e["rep"]?.fields["genre"] == "Technical report")207 #expect(e["rep"]?.fields["publisher"] == "Bank of Canada")208 #expect(e["rep"]?.fields["number"] == "2020-5")209 #expect(e["chap"]?.type == "chapter")210 #expect(e["chap"]?.year == nil)211 #expect(e["chap"]?.fields[ReferenceField.issuedLiteral] == "forthcoming") // not turned into a number212 #expect(e["chap"]?.fields["collection-title"] == "Lecture Notes")213 #expect(e["chap"]?.fields["collection-number"] == "7")214 #expect(e["arx"]?.type == "document")215 #expect(e["arx"]?.fields["URL"] == "https://arxiv.org/abs/2301.00001")216 #expect(e["arx"]?.fields["eprint"] == "2301.00001")217 #expect(e["arx"]?.fields["eprinttype"] == "arXiv")218 #expect(e["art"]?.type == "graphic")219 #expect([e["art"]?.year, e["art"]?.month] == [1889, 6])220 #expect(e["art"]?.fields[ReferenceField.rawDate] == "1889-06~") // approximate date kept221 #expect(e["u"]?.type == "document")222 #expect(e["u"]?.fields[ReferenceField.entryType] == "unknowntype")223 #expect(r.issues.count == 1)224 #expect(r.issues[0].contains("Line 14") && r.issues[0].contains("@unknowntype"))225 }226227 @Test func errorsAreReportedWithLineNumbersAndParsingResumes() {228 let r = BibTeXParser.parse(#"""229 @article{good1, title = {One}, year = 2001}230 @article{broken, title = {Two} year = 2002}231 @article{good2, title = {Three}, year = {2003}, month = sept, month = oct, note = undefinedmacro}232 @book{good1, title = {Duplicate}}233 @book{nokey2, title = {Unclosed {brace}, year = 2004}234 @book{good3, title = {Four}}235 """#)236 #expect(r.entries.map(\.key) == ["good1", "broken", "good2", "nokey2", "good3"])237 #expect(r.entries[3].title == "Unclosed brace, year = 2004")238 #expect(r.entries[1].title == "Two") // what was read before the error is kept239 #expect(r.entries[1].year == nil)240 #expect(r.entries[2].month == 9)241 #expect(r.entries[2].fields["note"] == "undefinedmacro")242 #expect(r.issues.contains { $0.hasPrefix("Line 2:") && $0.contains("missing comma after field \"title\"") })243 #expect(r.issues.contains { $0.hasPrefix("Line 3:") && $0.contains("\"month\" is given twice") })244 #expect(r.issues.contains { $0.hasPrefix("Line 3:") && $0.contains("undefined macro \"undefinedmacro\"") })245 #expect(r.issues.contains { $0.hasPrefix("Line 4:") && $0.contains("duplicate key \"good1\"") })246 #expect(r.issues.contains { $0.hasPrefix("Line 6:") && $0.contains("entry \"nokey2\": missing comma") })247 let open = BibTeXParser.parse("@book{a, title = {Fine}}\n@book{x,\n title = {Never closed")248 #expect(open.entries.map(\.key) == ["a"])249 #expect(open.issues.count == 1 && open.issues[0].hasPrefix("Line 3:") && open.issues[0].contains("unbalanced braces"))250 // Nothing is made up for an empty or hopeless input.251 #expect(BibTeXParser.parse("").entries.isEmpty)252 #expect(BibTeXParser.parse("just text, no entries @ all").entries.isEmpty)253 let noKey = BibTeXParser.parse("@book{, author = {Roy, Anne}, title = {The Long Winter}, year = 1999}")254 #expect(noKey.entries.map(\.key) == ["roy1999long"])255 #expect(noKey.issues.count == 1)256 }257258 @Test func writerRoundTrip() {259 let first = BibTeXParser.parse(Self.sample).entries260 let written = BibTeXWriter.write(first)261 let second = BibTeXParser.parse(written)262 #expect(second.issues.isEmpty, "\(second.issues)")263 #expect(second.entries == first)264 #expect(written.contains("mycustomfield = {kept as is}"))265 #expect(written.contains("month = jul"))266 #expect(written.contains("author = {Engle, Robert F. and Bollerslev, Tim and de la Fuente, Ángel and others}"))267 #expect(written.contains("{World Health Organization}") == false)268 #expect(written.contains("author = {Vaswani, Ashish and {Google Brain Team}}"))269 // Entries built in code, with awkward content, survive too.270 let odd = [271 BibEntry(key: "odd1", type: "article-magazine", authors: [org("Barnes and Noble, Inc."), person("O'Neil", "Shaquille, Jr."), .others],272 editors: [person("De La Cruz")], year: 2020, month: 12, day: 31,273 fields: ["title": "NASA & the iPhone: 100% {weird} $x_1$ — “quoted” <i>Homo sapiens</i>", "container-title": "Wired", "issue": "5", "number": "N-12",274 "page": "12–14, 20", "URL": "https://example.org/a_b?c=d%20e#f", "DOI": "10.1000/xyz_(123)", "abstract": "Line one.\nLine two.",275 "translator": "Goldstein, Ann and {Some Collective}", "accessed": "2021-01-02", "x-custom": "value", "keyword": "a, b; c"]),276 BibEntry(key: "odd2", type: "speech", fields: ["title": "A speech", ReferenceField.issuedLiteral: "in press"]),277 BibEntry(key: "odd3", type: "thesis", authors: [person("Roy", "A.")], year: 2020, fields: ["title": "T", "genre": "Doctoral dissertation", "publisher": "UQO"]),278 BibEntry(key: "odd4", type: "report", year: 1850, fields: ["title": "R", ReferenceField.rawDate: "1850~", "collection-title": "Series", "number": "4"]),279 ]280 let again = BibTeXParser.parse(BibTeXWriter.write(odd))281 #expect(again.issues.isEmpty, "\(again.issues)")282 #expect(again.entries == odd)283 #expect(BibTeXWriter.write(odd).contains("title = {{NASA} \\& the {iPhone}:") == false) // titles with markup or math are not case-protected284 #expect(BibTeXWriter.write([BibEntry(key: "k", type: "book", fields: ["title": "GARCH models at NASA"])]).contains("title = {{GARCH} models at {NASA}}"))285 }286}287