SPB Git

Repository Public

ImportCorpusTests.swift 14.8 KB 247 lines swift

1// ORIX Docs2// File: Packages/ORIXDocsKit/Tests/DocDOCXTests/ImportCorpusTests.swift3// Description: Import fidelity on the tool-generated corpus (pandoc and LibreOffice files): every assertion is on the resulting4//              model and is derived from the fixture sources in Corpus/sources.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import Testing10import DocModel11import DocContainer12@testable import DocDOCX1314@Suite("DOCX import — pandoc and LibreOffice corpus")15struct ImportCorpusTests {16    func load(_ name: String) throws -> ImportResult { try DocxImporter.importDocument(try Corpus.data(name)) }1718    func run(_ doc: DocumentModel, _ text: String) -> Run? {19        for p in doc.bodyParagraphs { if let r = p.runs.first(where: { $0.text == text }) { return r } }20        return nil21    }2223    @Test func corpusIsPresent() {24        #expect(Corpus.toolFiles.filter { $0.hasPrefix("pandoc-") }.count == 8)25        #expect(Corpus.toolFiles.filter { $0.hasPrefix("libreoffice-") }.count == 2)26    }2728    @Test func pandocBasicTextAndRuns() throws {29        let doc = try load("pandoc-basic.docx").document30        #expect(doc.meta.title == "Basic formatting fixture")31        #expect(doc.meta.author == "ORIX Docs corpus")32        #expect(doc.meta.keywords == ["corpus", "formatting"])33        #expect(doc.bodyParagraphs.first?.style == StyleID.title)34        #expect(run(doc, "emphasis")?.props.italic == true)35        #expect(run(doc, "strong text")?.props.bold == true)36        #expect(run(doc, "struck text")?.props.strikethrough == true)37        #expect(run(doc, "Small Caps")?.props.smallCaps == true)38        let para = try #require(doc.firstParagraph(containing: "A paragraph with"))39        #expect(para.runs.first { $0.text == "2" && $0.props.vertical == .subscript } != nil)40        #expect(para.runs.first { $0.text == "2" && $0.props.vertical == .superscript } != nil)41        #expect(para.visibleText == "A paragraph with emphasis, strong text, struck text, inline code, H2O and x2, a link and Small Caps.")42        #expect(run(doc, "a link")?.props.link == "https://example.org/page?x=1&y=2")43        #expect(run(doc, "inline code")?.charStyle == "VerbatimChar")44    }4546    @Test func pandocBasicStructure() throws {47        let doc = try load("pandoc-basic.docx").document48        let outline = Numbering(doc).outline49        #expect(outline.map(\.level) == [1, 2, 3])50        #expect(outline.map(\.text) == ["First heading", "Second level", "Third level"])51        let h1 = try #require(doc.firstParagraph(containing: "First heading"))52        #expect(h1.style == StyleID.heading(1))53        // The heading anchor pandoc writes becomes a bookmark, and the internal link points to it.54        #expect(h1.runs.contains { $0.inline == .bookmark("first-heading") })55        #expect(run(doc, "internal link")?.props.link == "#first-heading")56        #expect(Numbering(doc).bookmarks["first-heading"] == h1.id)57        // Forced line break and code block lines.58        #expect(doc.firstParagraph(containing: "hard")?.plainText == "A line with a hard\u{2028}line break inside.")59        let code = try #require(doc.bodyParagraphs.first { $0.style == "SourceCode" })60        #expect(code.visibleText == "def f(x):\u{2028}    return x + 1")61        #expect(doc.firstParagraph(containing: "block quote")?.style == "BlockText")62        #expect(doc.styles.style("BlockText") != nil)63    }6465    @Test func pandocListsLevelsAndLabels() throws {66        let doc = try load("pandoc-lists.docx").document67        let items = doc.bodyParagraphs.filter { $0.list != nil }68        #expect(items.map(\.visibleText) == ["Alpha", "Beta", "Beta one", "Beta two", "Deep item", "Gamma", "First", "Second", "Second a", "Second b", "Third",69                                             "Starts at five", "Six", "Roman one", "Roman two"])70        #expect(items.map { $0.list!.level } == [0, 0, 1, 1, 2, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0])71        let labels = Numbering(doc).listLabels72        let l = items.map { labels[$0.id] ?? "" }73        // Bullets: one glyph, never a number.74        for bullet in l[0..<6] { #expect(bullet.count == 1 && !bullet.contains(where: \.isNumber)) }75        #expect(Array(l[6...]) == ["1.", "2.", "a.", "b.", "3.", "5.", "6.", "i.", "ii."])76        // pandoc gives every nesting level its own w:num; the three top-level bullets share one definition,77        // and the list restarting at five has its own.78        #expect(Set([items[0], items[1], items[5]].map { $0.list!.listID }).count == 1)79        #expect(items[6].list!.listID == items[10].list!.listID)80        #expect(items[6].list!.listID != items[11].list!.listID)81        #expect(doc.firstParagraph(containing: "Text between lists.")?.list == nil)82    }8384    @Test func pandocTables() throws {85        let doc = try load("pandoc-tables.docx").document86        #expect(doc.tables.count == 2)87        let t = doc.tables[0]88        #expect(t.columnCount == 3)89        #expect(t.rows.count == 4)90        #expect(t.rows.map(\.isHeader) == [true, false, false, false])91        #expect(t.rows.map { $0.cells.map(\.plainText) } == [["Left", "Center", "Right"], ["a1", "b1", "1.5"], ["a2", "b2", "22.75"], ["a3", "b3", "300"]])92        #expect(t.rows[1].cells.map { $0.paragraphs[0].props.alignment } == [.left, .center, .right])93        #expect(t.rows[3].cells[1].paragraphs[0].runs.first?.props.bold == true)94        #expect(t.rows.allSatisfy { $0.cells.allSatisfy { $0.colSpan == 1 && $0.vMerge == .none } })95        let t2 = doc.tables[1]96        #expect(t2.columnCount == 2 && t2.rows.count == 3)97        #expect(t2.rows[1].cells[1].paragraphs.map(\.visibleText) == ["first line", "second para"])98        #expect(doc.firstParagraph(containing: "Sample pipe table") != nil)99        #expect(doc.bodyLines.last == "After the tables.")100    }101102    @Test func pandocFootnotes() throws {103        let doc = try load("pandoc-footnotes.docx").document104        let numbering = Numbering(doc)105        #expect(numbering.footnoteOrder.count == 3)106        #expect(doc.notes.count == 3)107        let notes = numbering.footnoteOrder.map { doc.notes[$0]! }108        #expect(notes.allSatisfy { $0.kind == .footnote })109        #expect(notes[0].paragraphs.map(\.visibleText) == ["The first footnote."])110        #expect(notes[1].paragraphs.map(\.visibleText) == ["A note with two paragraphs.", "Second paragraph of the note."])111        #expect(notes[2].paragraphs.map(\.visibleText) == ["An inline note with emphasis."])112        #expect(notes[2].paragraphs[0].runs.first { $0.text == "emphasis" }?.props.italic == true)113        // The reference sits exactly where the mark is in the source.114        let p = try #require(doc.firstParagraph(containing: "Here is a statement."))115        #expect(p.plainText == "Here is a statement.\u{FFFC} And another one with a longer note.\u{FFFC}")116        #expect(p.visibleText == "Here is a statement. And another one with a longer note.")117    }118119    @Test func pandocMathBecomesLatex() throws {120        let r = try load("pandoc-math.docx")121        let equations: [Equation] = r.document.inlines.compactMap { if case .equation(let e) = $0 { return e } else { return nil } }122        #expect(equations == [123            Equation(latex: "E=mc^{2}", display: false),124            Equation(latex: "\\alpha+\\beta\\leq\\gamma", display: false),125            Equation(latex: "\\frac{a+b}{c}=\\sqrt{x^{2}+y^{2}}", display: true),126            Equation(latex: "\\sum_{i=1}^{n}x_{i}=\\int_{0}^{1}f(t)\\,dt", display: true),127            Equation(latex: "\\begin{pmatrix}a&b\\\\c&d\\end{pmatrix}", display: false),128        ])129        #expect(!r.warnings.contains { $0.feature == "Equations" })130        #expect(r.document.firstParagraph(containing: "Inline math")?.visibleText == "Inline math  and  in a sentence.")131    }132133    @Test func pandocImages() throws {134        let doc = try load("pandoc-images.docx").document135        let images: [ImageObject] = doc.inlines.compactMap { if case .image(let i) = $0 { return i } else { return nil } }136        #expect(images.count == 2)137        #expect(doc.assets.count == 2)138        let red = try Data(contentsOf: Corpus.directory.appendingPathComponent("sources/pixel-red.png"))139        let blue = try Data(contentsOf: Corpus.directory.appendingPathComponent("sources/pixel-blue.png"))140        #expect(doc.assets[images[0].asset]?.data == red)141        #expect(doc.assets[images[1].asset]?.data == blue)142        #expect(doc.assets[images[0].asset]?.mimeType == "image/png")143        #expect(images[0].altText == "A red rectangle")144        #expect(images[1].altText == "blue square")145        #expect(abs(images[0].width - 72) < 0.01 && abs(images[0].height - 48) < 0.1)146        #expect(abs(images[1].width - 18) < 0.01)147        #expect(images.allSatisfy { $0.placement == .inline })148    }149150    @Test func pandocUnicodeIsIntactAndCountedInUTF16() throws {151        let doc = try load("pandoc-unicode.docx").document152        let p = try #require(doc.firstParagraph(containing: "Emoji"))153        #expect(p.visibleText == "Emoji 👩‍👩‍👧‍👦 and 🇨🇦 flags, combining e\u{301} marks: é, astral 𝒳 𝔘, CJK 日本語のテキスト, Arabic مرحبا بالعالم, Hebrew שלום.")154        #expect(p.length == p.plainText.utf16.count)155        #expect(p.length > p.plainText.count)156        let q = try #require(doc.firstParagraph(containing: "curly"))157        #expect(q.visibleText == "Non\u{2011}breaking hyphen, soft\u{AD}hyphen, “curly quotes”, em—dash, ellipsis…")158        #expect(Numbering(doc).outline.first?.text == "Unicode — ünïcödé")159    }160161    @Test func pandocTableOfContentsBecomesOneBlock() throws {162        let doc = try load("pandoc-toc.docx").document163        let tocs: [TOCBlock] = doc.body.compactMap { if case .toc(let t) = $0 { return t } else { return nil } }164        #expect(tocs.count == 1)165        #expect(tocs.first?.kind == .contents)166        #expect(tocs.first?.maxLevel == 2)167        #expect(tocs.first?.title == "Table of Contents")168        // Neither the title nor cached entries are left behind as paragraphs.169        #expect(doc.bodyLines == ["Contents fixture", "Introduction", "Text of the introduction.", "Background", "Text.", "Methods", "Text.", "Data", "Sources", "Text."])170    }171172    @Test func pandocPartsArePreservedOpaque() throws {173        let r = try load("pandoc-basic.docx")174        #expect(r.document.preservedParts.keys.sorted() == ["docProps/app.xml", "docProps/custom.xml", "word/fontTable.xml", "word/theme/theme1.xml", "word/webSettings.xml"])175        #expect(r.warnings.contains { $0.feature == "Preserved parts" && $0.severity == .info })176        // The only thing lost from a pandoc file is the "contextual spacing" flag of its Title style, and it is reported.177        #expect(r.warnings.filter { $0.severity == .dropped }.map(\.feature) == ["Other paragraph formatting"])178        #expect(!r.warnings.contains { $0.severity == .approximated })179    }180181    @Test func libreOfficeMergedCellsAndCharacterFormatting() throws {182        let doc = try load("libreoffice-merged.docx").document183        let t = try #require(doc.tables.first)184        #expect(t.columnCount == 3)185        #expect(t.rows.map { $0.cells.map(\.colSpan) } == [[2, 1], [1, 1, 1], [1, 1, 1], [1, 2]])186        #expect(t.rows.map { $0.cells.map(\.vMerge) } == [[.none, .none], [.restart, .none, .none], [.continue, .none, .none], [.none, .none]])187        #expect(t.rows[1].cells[0].plainText == "Tall")188        #expect(t.rows[3].cells[1].plainText == "Wide body")189        #expect(run(doc, "bold")?.props.bold == true)190        #expect(run(doc, "italic")?.props.italic == true)191        #expect(run(doc, "underline")?.props.underline == true)192        #expect(run(doc, "red")?.props.color == RGBA(hex: 0xFF0000))193        #expect(run(doc, "highlight")?.props.highlight == RGBA(hex: 0xFFFF00))194        #expect(doc.firstParagraph(containing: "Centered paragraph.")?.props.alignment == .center)195        let right = try #require(doc.firstParagraph(containing: "Right aligned"))196        #expect(right.props.alignment == .right)197        #expect(abs((right.props.indentLeft ?? 0) - 72) < 0.5)198        let labels = Numbering(doc).listLabels199        #expect(doc.bodyParagraphs.filter { $0.list != nil }.map { labels[$0.id] ?? "" }.prefix(2) == ["1.", "2."])200        #expect(doc.firstParagraph(containing: "Nested bullet")?.list?.level == 1)201    }202203    @Test func libreOfficePageSetupSectionsHeadersFooters() throws {204        let doc = try load("libreoffice-pagesetup.docx").document205        let first = doc.pageSetup206        #expect(first.paper == .a4)207        #expect(!first.landscape)208        #expect(abs(first.margins.left - 85.05) < 0.1)      // 3 cm209        #expect(abs(first.margins.right - 42.5) < 0.1)      // 1.5 cm210        #expect(first.columns == 1)211        #expect(first.header.default.map(\.visibleText) == ["Running header text"])212        #expect(first.header.default.first?.props.alignment == .center)213        let footer = try #require(first.footer.default.first)214        #expect(footer.runs.compactMap(\.inline) == [.field(.pageNumber), .field(.pageCount)])215        #expect(footer.visibleText == "Page  of ")216217        let breaks: [SectionBreak] = doc.body.compactMap { if case .sectionBreak(let s) = $0 { return s } else { return nil } }218        #expect(breaks.count == 3)219        // The break before the two-column text STARTS the two-column section.220        #expect(breaks[0].kind == .continuous && breaks[0].setup.columns == 2)221        #expect(abs(breaks[0].setup.columnGap - 28.35) < 0.1)   // 1 cm222        #expect(breaks[1].kind == .continuous && breaks[1].setup.columns == 1)223        #expect(breaks[2].kind == .nextPage && breaks[2].setup.landscape && breaks[2].setup.paper == .a4)224        #expect(abs(breaks[2].setup.pageWidth - 841.89) < 0.01)225        // Position of the breaks relative to the text.226        let order = doc.body.map { b -> String in227            switch b {228            case .paragraph(let p): return p.visibleText229            case .table: return "<table>"230            case .sectionBreak: return "<section>"231            default: return "<other>"232            }233        }.filter { !$0.isEmpty }234        #expect(order == ["Page setup fixture", "First section paragraph on A4 portrait with asymmetric margins.", "<section>", "Two column section, first paragraph.",235                          "Two column section, second paragraph.", "<section>", "Back to one column.", "<table>", "<section>", "Landscape page paragraph."])236        let t = try #require(doc.tables.first)237        #expect(t.rows.map { $0.cells.map(\.colSpan) } == [[2, 1], [1, 1, 1], [1, 1, 1]])238        #expect(t.rows[1].cells[0].vMerge == .restart && t.rows[2].cells[0].vMerge == .continue)239        #expect(t.rows[0].cells[0].shading == RGBA(hex: 0xDDEEFF))240        #expect(t.rows[0].cells[0].borderTop != nil)241        // Paragraph ids stay unique even though three sections share one header.242        var ids: [String] = []243        doc.forEachParagraphEverywhere { p, _, _ in ids.append(p.id) }244        #expect(Set(ids).count == ids.count)245    }246}247
Files main
Menu

Theme