Repository Public
1// ORIX Docs2// File: Packages/ORIXDocsKit/Tests/DocDOCXTests/ImportCorpusTests.swift3// Description: Import fidelity on the tool-generated corpus (pandoc and LibreOffice files): every assertion is on the resulting4// model and is derived from the fixture sources in Corpus/sources.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import Testing10import DocModel11import DocContainer12@testable import DocDOCX1314@Suite("DOCX import — pandoc and LibreOffice corpus")15struct ImportCorpusTests {16 func load(_ name: String) throws -> ImportResult { try DocxImporter.importDocument(try Corpus.data(name)) }1718 func run(_ doc: DocumentModel, _ text: String) -> Run? {19 for p in doc.bodyParagraphs { if let r = p.runs.first(where: { $0.text == text }) { return r } }20 return nil21 }2223 @Test func corpusIsPresent() {24 #expect(Corpus.toolFiles.filter { $0.hasPrefix("pandoc-") }.count == 8)25 #expect(Corpus.toolFiles.filter { $0.hasPrefix("libreoffice-") }.count == 2)26 }2728 @Test func pandocBasicTextAndRuns() throws {29 let doc = try load("pandoc-basic.docx").document30 #expect(doc.meta.title == "Basic formatting fixture")31 #expect(doc.meta.author == "ORIX Docs corpus")32 #expect(doc.meta.keywords == ["corpus", "formatting"])33 #expect(doc.bodyParagraphs.first?.style == StyleID.title)34 #expect(run(doc, "emphasis")?.props.italic == true)35 #expect(run(doc, "strong text")?.props.bold == true)36 #expect(run(doc, "struck text")?.props.strikethrough == true)37 #expect(run(doc, "Small Caps")?.props.smallCaps == true)38 let para = try #require(doc.firstParagraph(containing: "A paragraph with"))39 #expect(para.runs.first { $0.text == "2" && $0.props.vertical == .subscript } != nil)40 #expect(para.runs.first { $0.text == "2" && $0.props.vertical == .superscript } != nil)41 #expect(para.visibleText == "A paragraph with emphasis, strong text, struck text, inline code, H2O and x2, a link and Small Caps.")42 #expect(run(doc, "a link")?.props.link == "https://example.org/page?x=1&y=2")43 #expect(run(doc, "inline code")?.charStyle == "VerbatimChar")44 }4546 @Test func pandocBasicStructure() throws {47 let doc = try load("pandoc-basic.docx").document48 let outline = Numbering(doc).outline49 #expect(outline.map(\.level) == [1, 2, 3])50 #expect(outline.map(\.text) == ["First heading", "Second level", "Third level"])51 let h1 = try #require(doc.firstParagraph(containing: "First heading"))52 #expect(h1.style == StyleID.heading(1))53 // The heading anchor pandoc writes becomes a bookmark, and the internal link points to it.54 #expect(h1.runs.contains { $0.inline == .bookmark("first-heading") })55 #expect(run(doc, "internal link")?.props.link == "#first-heading")56 #expect(Numbering(doc).bookmarks["first-heading"] == h1.id)57 // Forced line break and code block lines.58 #expect(doc.firstParagraph(containing: "hard")?.plainText == "A line with a hard\u{2028}line break inside.")59 let code = try #require(doc.bodyParagraphs.first { $0.style == "SourceCode" })60 #expect(code.visibleText == "def f(x):\u{2028} return x + 1")61 #expect(doc.firstParagraph(containing: "block quote")?.style == "BlockText")62 #expect(doc.styles.style("BlockText") != nil)63 }6465 @Test func pandocListsLevelsAndLabels() throws {66 let doc = try load("pandoc-lists.docx").document67 let items = doc.bodyParagraphs.filter { $0.list != nil }68 #expect(items.map(\.visibleText) == ["Alpha", "Beta", "Beta one", "Beta two", "Deep item", "Gamma", "First", "Second", "Second a", "Second b", "Third",69 "Starts at five", "Six", "Roman one", "Roman two"])70 #expect(items.map { $0.list!.level } == [0, 0, 1, 1, 2, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0])71 let labels = Numbering(doc).listLabels72 let l = items.map { labels[$0.id] ?? "" }73 // Bullets: one glyph, never a number.74 for bullet in l[0..<6] { #expect(bullet.count == 1 && !bullet.contains(where: \.isNumber)) }75 #expect(Array(l[6...]) == ["1.", "2.", "a.", "b.", "3.", "5.", "6.", "i.", "ii."])76 // pandoc gives every nesting level its own w:num; the three top-level bullets share one definition,77 // and the list restarting at five has its own.78 #expect(Set([items[0], items[1], items[5]].map { $0.list!.listID }).count == 1)79 #expect(items[6].list!.listID == items[10].list!.listID)80 #expect(items[6].list!.listID != items[11].list!.listID)81 #expect(doc.firstParagraph(containing: "Text between lists.")?.list == nil)82 }8384 @Test func pandocTables() throws {85 let doc = try load("pandoc-tables.docx").document86 #expect(doc.tables.count == 2)87 let t = doc.tables[0]88 #expect(t.columnCount == 3)89 #expect(t.rows.count == 4)90 #expect(t.rows.map(\.isHeader) == [true, false, false, false])91 #expect(t.rows.map { $0.cells.map(\.plainText) } == [["Left", "Center", "Right"], ["a1", "b1", "1.5"], ["a2", "b2", "22.75"], ["a3", "b3", "300"]])92 #expect(t.rows[1].cells.map { $0.paragraphs[0].props.alignment } == [.left, .center, .right])93 #expect(t.rows[3].cells[1].paragraphs[0].runs.first?.props.bold == true)94 #expect(t.rows.allSatisfy { $0.cells.allSatisfy { $0.colSpan == 1 && $0.vMerge == .none } })95 let t2 = doc.tables[1]96 #expect(t2.columnCount == 2 && t2.rows.count == 3)97 #expect(t2.rows[1].cells[1].paragraphs.map(\.visibleText) == ["first line", "second para"])98 #expect(doc.firstParagraph(containing: "Sample pipe table") != nil)99 #expect(doc.bodyLines.last == "After the tables.")100 }101102 @Test func pandocFootnotes() throws {103 let doc = try load("pandoc-footnotes.docx").document104 let numbering = Numbering(doc)105 #expect(numbering.footnoteOrder.count == 3)106 #expect(doc.notes.count == 3)107 let notes = numbering.footnoteOrder.map { doc.notes[$0]! }108 #expect(notes.allSatisfy { $0.kind == .footnote })109 #expect(notes[0].paragraphs.map(\.visibleText) == ["The first footnote."])110 #expect(notes[1].paragraphs.map(\.visibleText) == ["A note with two paragraphs.", "Second paragraph of the note."])111 #expect(notes[2].paragraphs.map(\.visibleText) == ["An inline note with emphasis."])112 #expect(notes[2].paragraphs[0].runs.first { $0.text == "emphasis" }?.props.italic == true)113 // The reference sits exactly where the mark is in the source.114 let p = try #require(doc.firstParagraph(containing: "Here is a statement."))115 #expect(p.plainText == "Here is a statement.\u{FFFC} And another one with a longer note.\u{FFFC}")116 #expect(p.visibleText == "Here is a statement. And another one with a longer note.")117 }118119 @Test func pandocMathBecomesLatex() throws {120 let r = try load("pandoc-math.docx")121 let equations: [Equation] = r.document.inlines.compactMap { if case .equation(let e) = $0 { return e } else { return nil } }122 #expect(equations == [123 Equation(latex: "E=mc^{2}", display: false),124 Equation(latex: "\\alpha+\\beta\\leq\\gamma", display: false),125 Equation(latex: "\\frac{a+b}{c}=\\sqrt{x^{2}+y^{2}}", display: true),126 Equation(latex: "\\sum_{i=1}^{n}x_{i}=\\int_{0}^{1}f(t)\\,dt", display: true),127 Equation(latex: "\\begin{pmatrix}a&b\\\\c&d\\end{pmatrix}", display: false),128 ])129 #expect(!r.warnings.contains { $0.feature == "Equations" })130 #expect(r.document.firstParagraph(containing: "Inline math")?.visibleText == "Inline math and in a sentence.")131 }132133 @Test func pandocImages() throws {134 let doc = try load("pandoc-images.docx").document135 let images: [ImageObject] = doc.inlines.compactMap { if case .image(let i) = $0 { return i } else { return nil } }136 #expect(images.count == 2)137 #expect(doc.assets.count == 2)138 let red = try Data(contentsOf: Corpus.directory.appendingPathComponent("sources/pixel-red.png"))139 let blue = try Data(contentsOf: Corpus.directory.appendingPathComponent("sources/pixel-blue.png"))140 #expect(doc.assets[images[0].asset]?.data == red)141 #expect(doc.assets[images[1].asset]?.data == blue)142 #expect(doc.assets[images[0].asset]?.mimeType == "image/png")143 #expect(images[0].altText == "A red rectangle")144 #expect(images[1].altText == "blue square")145 #expect(abs(images[0].width - 72) < 0.01 && abs(images[0].height - 48) < 0.1)146 #expect(abs(images[1].width - 18) < 0.01)147 #expect(images.allSatisfy { $0.placement == .inline })148 }149150 @Test func pandocUnicodeIsIntactAndCountedInUTF16() throws {151 let doc = try load("pandoc-unicode.docx").document152 let p = try #require(doc.firstParagraph(containing: "Emoji"))153 #expect(p.visibleText == "Emoji 👩👩👧👦 and 🇨🇦 flags, combining e\u{301} marks: é, astral 𝒳 𝔘, CJK 日本語のテキスト, Arabic مرحبا بالعالم, Hebrew שלום.")154 #expect(p.length == p.plainText.utf16.count)155 #expect(p.length > p.plainText.count)156 let q = try #require(doc.firstParagraph(containing: "curly"))157 #expect(q.visibleText == "Non\u{2011}breaking hyphen, soft\u{AD}hyphen, “curly quotes”, em—dash, ellipsis…")158 #expect(Numbering(doc).outline.first?.text == "Unicode — ünïcödé")159 }160161 @Test func pandocTableOfContentsBecomesOneBlock() throws {162 let doc = try load("pandoc-toc.docx").document163 let tocs: [TOCBlock] = doc.body.compactMap { if case .toc(let t) = $0 { return t } else { return nil } }164 #expect(tocs.count == 1)165 #expect(tocs.first?.kind == .contents)166 #expect(tocs.first?.maxLevel == 2)167 #expect(tocs.first?.title == "Table of Contents")168 // Neither the title nor cached entries are left behind as paragraphs.169 #expect(doc.bodyLines == ["Contents fixture", "Introduction", "Text of the introduction.", "Background", "Text.", "Methods", "Text.", "Data", "Sources", "Text."])170 }171172 @Test func pandocPartsArePreservedOpaque() throws {173 let r = try load("pandoc-basic.docx")174 #expect(r.document.preservedParts.keys.sorted() == ["docProps/app.xml", "docProps/custom.xml", "word/fontTable.xml", "word/theme/theme1.xml", "word/webSettings.xml"])175 #expect(r.warnings.contains { $0.feature == "Preserved parts" && $0.severity == .info })176 // The only thing lost from a pandoc file is the "contextual spacing" flag of its Title style, and it is reported.177 #expect(r.warnings.filter { $0.severity == .dropped }.map(\.feature) == ["Other paragraph formatting"])178 #expect(!r.warnings.contains { $0.severity == .approximated })179 }180181 @Test func libreOfficeMergedCellsAndCharacterFormatting() throws {182 let doc = try load("libreoffice-merged.docx").document183 let t = try #require(doc.tables.first)184 #expect(t.columnCount == 3)185 #expect(t.rows.map { $0.cells.map(\.colSpan) } == [[2, 1], [1, 1, 1], [1, 1, 1], [1, 2]])186 #expect(t.rows.map { $0.cells.map(\.vMerge) } == [[.none, .none], [.restart, .none, .none], [.continue, .none, .none], [.none, .none]])187 #expect(t.rows[1].cells[0].plainText == "Tall")188 #expect(t.rows[3].cells[1].plainText == "Wide body")189 #expect(run(doc, "bold")?.props.bold == true)190 #expect(run(doc, "italic")?.props.italic == true)191 #expect(run(doc, "underline")?.props.underline == true)192 #expect(run(doc, "red")?.props.color == RGBA(hex: 0xFF0000))193 #expect(run(doc, "highlight")?.props.highlight == RGBA(hex: 0xFFFF00))194 #expect(doc.firstParagraph(containing: "Centered paragraph.")?.props.alignment == .center)195 let right = try #require(doc.firstParagraph(containing: "Right aligned"))196 #expect(right.props.alignment == .right)197 #expect(abs((right.props.indentLeft ?? 0) - 72) < 0.5)198 let labels = Numbering(doc).listLabels199 #expect(doc.bodyParagraphs.filter { $0.list != nil }.map { labels[$0.id] ?? "" }.prefix(2) == ["1.", "2."])200 #expect(doc.firstParagraph(containing: "Nested bullet")?.list?.level == 1)201 }202203 @Test func libreOfficePageSetupSectionsHeadersFooters() throws {204 let doc = try load("libreoffice-pagesetup.docx").document205 let first = doc.pageSetup206 #expect(first.paper == .a4)207 #expect(!first.landscape)208 #expect(abs(first.margins.left - 85.05) < 0.1) // 3 cm209 #expect(abs(first.margins.right - 42.5) < 0.1) // 1.5 cm210 #expect(first.columns == 1)211 #expect(first.header.default.map(\.visibleText) == ["Running header text"])212 #expect(first.header.default.first?.props.alignment == .center)213 let footer = try #require(first.footer.default.first)214 #expect(footer.runs.compactMap(\.inline) == [.field(.pageNumber), .field(.pageCount)])215 #expect(footer.visibleText == "Page of ")216217 let breaks: [SectionBreak] = doc.body.compactMap { if case .sectionBreak(let s) = $0 { return s } else { return nil } }218 #expect(breaks.count == 3)219 // The break before the two-column text STARTS the two-column section.220 #expect(breaks[0].kind == .continuous && breaks[0].setup.columns == 2)221 #expect(abs(breaks[0].setup.columnGap - 28.35) < 0.1) // 1 cm222 #expect(breaks[1].kind == .continuous && breaks[1].setup.columns == 1)223 #expect(breaks[2].kind == .nextPage && breaks[2].setup.landscape && breaks[2].setup.paper == .a4)224 #expect(abs(breaks[2].setup.pageWidth - 841.89) < 0.01)225 // Position of the breaks relative to the text.226 let order = doc.body.map { b -> String in227 switch b {228 case .paragraph(let p): return p.visibleText229 case .table: return "<table>"230 case .sectionBreak: return "<section>"231 default: return "<other>"232 }233 }.filter { !$0.isEmpty }234 #expect(order == ["Page setup fixture", "First section paragraph on A4 portrait with asymmetric margins.", "<section>", "Two column section, first paragraph.",235 "Two column section, second paragraph.", "<section>", "Back to one column.", "<table>", "<section>", "Landscape page paragraph."])236 let t = try #require(doc.tables.first)237 #expect(t.rows.map { $0.cells.map(\.colSpan) } == [[2, 1], [1, 1, 1], [1, 1, 1]])238 #expect(t.rows[1].cells[0].vMerge == .restart && t.rows[2].cells[0].vMerge == .continue)239 #expect(t.rows[0].cells[0].shading == RGBA(hex: 0xDDEEFF))240 #expect(t.rows[0].cells[0].borderTop != nil)241 // Paragraph ids stay unique even though three sections share one header.242 var ids: [String] = []243 doc.forEachParagraphEverywhere { p, _, _ in ids.append(p.id) }244 #expect(Set(ids).count == ids.count)245 }246}247