SPB Git

Repository Public

CompatibilityReportTests.swift 20.5 KB 338 lines swift

1// ORIX Docs2// File: Packages/ORIXDocsKit/Tests/DocDOCXTests/CompatibilityReportTests.swift3// Description: Runs the whole corpus through Import → Export → Reimport → Export → Reimport, measures what survives (per feature,4//              text integrity, agreement with pandoc / textutil / LibreOffice) and writes the DOCX compatibility report5//              (Markdown + JSON) when ORIX_DOCX_REPORT_DIR is set. Every number in the report is measured here.6//7// Author: Simon-Pierre Boucher8// Contact: contact@spboucher.ai9import Foundation10import Testing11import DocModel12import DocContainer13@testable import DocDOCX1415struct FeatureCensus: Codable, Equatable {16    var counts: [String: Int] = [:]1718    init(_ doc: DocumentModel) {19        func add(_ k: String, _ n: Int = 1) { counts[k, default: 0] += n }20        let headingStyles = Set(doc.styles.styles.filter { (doc.styles.styleProps($0.id).para.outlineLevel ?? 0) >= 1 }.map(\.id))21        func scan(_ p: Paragraph, body: Bool) {22            if body {23                add("paragraphs")24                if headingStyles.contains(p.style) || (p.props.outlineLevel ?? 0) >= 1 { add("headings") }25                if p.list != nil { add("list items") }26            }27            if p.markRevision != nil { add("tracked paragraph marks") }28            for r in p.runs {29                if r.revision?.kind == .insertion { add("tracked insertions") }30                if r.revision?.kind == .deletion { add("tracked deletions") }31                if r.formatRevision != nil { add("tracked format changes") }32                if r.props.link != nil { add("hyperlink runs") }33                if !r.comments.isEmpty { add("commented runs") }34                if r.props.bold == true { add("bold runs") }35                if r.props.italic == true { add("italic runs") }36                switch r.inline {37                case .image?: add("images")38                case .shape?: add("shapes")39                case .equation?: add("equations")40                case .note(let n)?: add(n.kind == .footnote ? "footnote references" : "endnote references")41                case .citation?: add("citations")42                case .crossRef?: add("cross-references")43                case .field?: add("fields")44                case .bookmark?: add("bookmarks")45                case nil: break46                }47            }48        }49        doc.forEachBodyParagraph { p, _ in scan(p, body: true) }50        for n in doc.notes.values { for p in n.paragraphs { scan(p, body: false) } }51        for s in doc.sectionSetups {52            for p in s.header.default + s.header.first + s.header.even { add("header paragraphs"); scan(p, body: false) }53            for p in s.footer.default + s.footer.first + s.footer.even { add("footer paragraphs"); scan(p, body: false) }54        }55        for b in doc.body {56            switch b {57            case .table(let t):58                add("tables")59                add("table rows", t.rows.count)60                for r in t.rows { for c in r.cells { add("table cells"); if c.colSpan > 1 || c.vMerge != .none { add("merged cells") } } }61            case .pageBreak: add("page and column breaks")62            case .sectionBreak: add("section breaks")63            case .toc: add("generated lists (TOC)")64            case .bibliography: add("bibliography blocks")65            case .paragraph: break66            }67        }68        add("notes", doc.notes.count)69        add("comments", doc.comments.count)70        add("comment replies", doc.comments.reduce(0) { $0 + $1.replies.count })71        add("styles", doc.styles.styles.count)72        add("list definitions", doc.lists.count)73        add("image assets", doc.assets.count)74        add("preserved parts", doc.preservedParts.count)75        add("bibliography entries", doc.bibliography.count)76        counts = counts.filter { $0.value > 0 }77    }78}7980struct DocumentReport: Codable {81    var name: String82    var source: String83    var bytes: Int84    var imported = false85    var importError: String?86    var importWarnings: [String: Int] = [:]87    var warningFeatures: [String] = []88    var exportWarnings: [String: Int] = [:]89    var census: [String: Int] = [:]90    var featuresKept: [String: Bool] = [:]91    var modelEqualAfterFirstCycle = false92    var modelEqualAfterSecondCycle = false93    var textIntegrityAfterTwoCycles: Double = 094    var wordsInDocument = 095    var importAgreementWithPandoc: Double?96    var pandocReadsExport: Bool?97    var pandocTextIntegrityOfExport: Double?98    var textutilReadsExport: Bool?99    var textutilTextIntegrityOfExport: Double?100    var libreOfficeConvertsExportToPDF: Bool?101}102103struct CompatibilityReport: Codable {104    var generated: String105    var pandocVersion: String?106    var libreOfficeVersion: String?107    var textutilAvailable: Bool108    var slowChecksRun: Bool109    var microsoftWordUsed = false110    var documents: [DocumentReport] = []111    var featureTotals: [String: [String: Int]] = [:]112    var totals: [String: Double] = [:]113}114115@Suite("DOCX corpus cycle and compatibility report", .serialized)116struct CompatibilityReportTests {117    func allWords(_ doc: DocumentModel) -> [String] {118        var words: [String] = []119        doc.forEachBodyParagraph { p, _ in words += wordSequence(p.visibleText) }120        let n = Numbering(doc)121        for id in n.footnoteOrder + n.endnoteOrder { for p in doc.notes[id]?.paragraphs ?? [] { words += wordSequence(p.visibleText) } }122        return words123    }124125    func severityCounts(_ w: [FormatWarning]) -> [String: Int] {126        var out: [String: Int] = [:]127        for x in w { out[x.severity.rawValue, default: 0] += x.count }128        return out129    }130131    func version(_ tool: String?, _ args: [String]) -> String? {132        guard let tool else { return nil }133        let r = Tool.run(tool, args, timeout: 60)134        return String(decoding: r.stdout, as: UTF8.self).split(separator: "\n").first.map(String.init)135    }136137    @Test func corpusCycle() throws {138        var sources: [(name: String, source: String, data: Data)] = []139        for name in Corpus.toolFiles { sources.append((name, name.hasPrefix("pandoc") ? "pandoc" : "LibreOffice", try Corpus.data(name))) }140        for (name, data) in try Hand.all() { sources.append((name, "hand-authored OOXML", data)) }141        var options = DocxExportOptions()142        options.citations = FixedFormatter()143        sources.append(("orix-rich.docx", "ORIX Docs model", try DocxExporter.export(richDocument(), options: options).data))144145        if let dump = ProcessInfo.processInfo.environment["ORIX_DOCX_DUMP_DIR"] {146            try FileManager.default.createDirectory(atPath: dump, withIntermediateDirectories: true)147            for s in sources where s.source != "pandoc" && s.source != "LibreOffice" { try s.data.write(to: URL(fileURLWithPath: dump).appendingPathComponent(s.name)) }148        }149150        let slow = Tool.slowTestsEnabled && Tool.soffice != nil151        var report = CompatibilityReport(generated: ISO8601DateFormatter().string(from: Date()), pandocVersion: version(Tool.pandoc, ["--version"]),152                                         libreOfficeVersion: slow ? version(Tool.soffice, ["--version"]) : nil, textutilAvailable: Tool.textutil != nil, slowChecksRun: slow)153        if Tool.pandoc == nil { recordSkip("pandoc missing: pandoc columns of the compatibility report are empty") }154        if !slow { recordSkip("LibreOffice PDF conversion not measured in the compatibility report (needs ORIX_SLOW_TESTS=1 and soffice)") }155156        for s in sources {157            var d = DocumentReport(name: s.name, source: s.source, bytes: s.data.count)158            let a: ImportResult159            do { a = try DocxImporter.importDocument(s.data) } catch {160                d.importError = "\(error)"161                report.documents.append(d)162                Issue.record("\(s.name) cannot be imported: \(error)")163                continue164            }165            d.imported = true166            d.importWarnings = severityCounts(a.warnings)167            d.warningFeatures = Array(Set(a.warnings.filter { $0.severity != .info }.map { "\($0.severity.rawValue): \($0.feature)" })).sorted()168            let censusA = FeatureCensus(a.document)169            d.census = censusA.counts170171            let export1 = try DocxExporter.export(a.document, options: options)172            d.exportWarnings = severityCounts(export1.warnings)173            let b = try DocxImporter.importDocument(export1.data).document174            let export2 = try DocxExporter.export(b, options: options)175            let c = try DocxImporter.importDocument(export2.data).document176            let censusB = FeatureCensus(b), censusC = FeatureCensus(c)177            for (feature, n) in censusA.counts { d.featuresKept[feature] = censusB.counts[feature] == n && censusC.counts[feature] == n }178            d.modelEqualAfterFirstCycle = normalized(a.document) == normalized(b)179            d.modelEqualAfterSecondCycle = normalized(b) == normalized(c)180            let words = allWords(a.document)181            d.wordsInDocument = words.count182            d.textIntegrityAfterTwoCycles = textIntegrity(reference: words, candidate: allWords(c))183184            if Tool.pandoc != nil {185                if let astWords = Tool.pandocWords(s.data) { d.importAgreementWithPandoc = textIntegrity(reference: astWords, candidate: words) }186                if let original = Tool.pandocText(s.data) {187                    if let ours = Tool.pandocText(export1.data) {188                        d.pandocReadsExport = true189                        d.pandocTextIntegrityOfExport = textIntegrity(reference: wordSequence(original), candidate: wordSequence(ours))190                    } else { d.pandocReadsExport = false }191                }192            }193            if Tool.textutil != nil, let original = Tool.textutilText(s.data) {194                if let ours = Tool.textutilText(export1.data) {195                    d.textutilReadsExport = true196                    d.textutilTextIntegrityOfExport = textIntegrity(reference: wordSequence(original), candidate: wordSequence(ours))197                } else { d.textutilReadsExport = false }198            }199            if slow { d.libreOfficeConvertsExportToPDF = Tool.sofficeConvert(export1.data, to: "pdf").map { $0.prefix(5) == Data("%PDF-".utf8) } ?? false }200            report.documents.append(d)201202            // The cycle itself is a hard requirement, not just a measurement.203            expectSameModel(a.document, b, "\(s.name), first cycle")204            expectSameModel(b, c, "\(s.name), second cycle")205            #expect(d.textIntegrityAfterTwoCycles == 1.0, "\(s.name): text integrity after two cycles")206            #expect(d.featuresKept.values.allSatisfy { $0 }, "\(s.name): features changed count: \(d.featuresKept.filter { !$0.value }.keys.sorted())")207            if let ok = d.pandocReadsExport { #expect(ok, "pandoc cannot read the export of \(s.name)") }208            if let ok = d.textutilReadsExport { #expect(ok, "textutil cannot read the export of \(s.name)") }209            if let ok = d.libreOfficeConvertsExportToPDF { #expect(ok, "LibreOffice cannot convert the export of \(s.name)") }210        }211212        // Totals.213        var features: [String: [String: Int]] = [:]214        for d in report.documents {215            for (f, kept) in d.featuresKept {216                features[f, default: ["documents": 0, "pass": 0, "fail": 0, "instances": 0]]["documents"]! += 1217                features[f]![kept ? "pass" : "fail"]! += 1218                features[f]!["instances"]! += d.census[f] ?? 0219            }220        }221        report.featureTotals = features222        let docs = report.documents223        func mean(_ xs: [Double]) -> Double { xs.isEmpty ? 0 : xs.reduce(0, +) / Double(xs.count) }224        func weighted(_ pick: (DocumentReport) -> Double?) -> Double {225            let rows = docs.compactMap { d in pick(d).map { ($0, Double(d.wordsInDocument)) } }226            let w = rows.reduce(0) { $0 + $1.1 }227            return w == 0 ? 0 : rows.reduce(0) { $0 + $1.0 * $1.1 } / w228        }229        report.totals = [230            "documents": Double(docs.count),231            "imported": Double(docs.filter(\.imported).count),232            "modelEqualAfterFirstCycle": Double(docs.filter(\.modelEqualAfterFirstCycle).count),233            "modelEqualAfterSecondCycle": Double(docs.filter(\.modelEqualAfterSecondCycle).count),234            "words": Double(docs.reduce(0) { $0 + $1.wordsInDocument }),235            "textIntegrityAfterTwoCyclesPercent": weighted { $0.textIntegrityAfterTwoCycles } * 100,236            "importAgreementWithPandocPercentMean": mean(docs.compactMap(\.importAgreementWithPandoc)) * 100,237            "pandocReadsExport": Double(docs.filter { $0.pandocReadsExport == true }.count),238            "pandocTextIntegrityOfExportPercentMean": mean(docs.compactMap(\.pandocTextIntegrityOfExport)) * 100,239            "textutilReadsExport": Double(docs.filter { $0.textutilReadsExport == true }.count),240            "textutilTextIntegrityOfExportPercentMean": mean(docs.compactMap(\.textutilTextIntegrityOfExport)) * 100,241            "libreOfficeConvertsExportToPDF": Double(docs.filter { $0.libreOfficeConvertsExportToPDF == true }.count),242            "importWarningsInfo": Double(docs.reduce(0) { $0 + ($1.importWarnings["info"] ?? 0) }),243            "importWarningsApproximated": Double(docs.reduce(0) { $0 + ($1.importWarnings["approximated"] ?? 0) }),244            "importWarningsDropped": Double(docs.reduce(0) { $0 + ($1.importWarnings["dropped"] ?? 0) }),245            "exportWarnings": Double(docs.reduce(0) { $0 + $1.exportWarnings.values.reduce(0, +) }),246            "featurePass": Double(features.values.reduce(0) { $0 + $1["pass"]! }),247            "featureFail": Double(features.values.reduce(0) { $0 + $1["fail"]! }),248        ]249        #expect(report.totals["imported"] == report.totals["documents"])250251        guard let dir = ProcessInfo.processInfo.environment["ORIX_DOCX_REPORT_DIR"] else {252            recordSkip("ORIX_DOCX_REPORT_DIR not set: compatibility report measured but not written")253            return254        }255        let enc = JSONEncoder()256        enc.outputFormatting = [.prettyPrinted, .sortedKeys]257        try FileManager.default.createDirectory(atPath: dir, withIntermediateDirectories: true)258        try enc.encode(report).write(to: URL(fileURLWithPath: dir).appendingPathComponent("docx-compatibility.json"))259        try markdown(report).write(to: URL(fileURLWithPath: dir).appendingPathComponent("docx-compatibility.md"), atomically: true, encoding: .utf8)260    }261262    func markdown(_ r: CompatibilityReport) -> String {263        func pct(_ v: Double?) -> String { v.map { String(format: "%.1f %%", $0 * 100) } ?? "n/a" }264        func yn(_ v: Bool?) -> String { v.map { $0 ? "yes" : "NO" } ?? "n/a" }265        func n(_ key: String) -> Int { Int(r.totals[key] ?? 0) }266        var s = """267        <!--268        ORIX Docs269        File: docs/reports/docx-compatibility.md270        Description: Measured DOCX compatibility of the ORIX Docs engine on the reference corpus. Generated file — do not edit; run Scripts/docx-compat.sh.271        Author: Simon-Pierre Boucher272        Contact: contact@spboucher.ai273        -->274        # DOCX compatibility report (measured)275276        Generated \(r.generated) by `Scripts/docx-compat.sh` (test `CompatibilityReportTests.corpusCycle`). Every number below was measured during that run.277278        * pandoc: \(r.pandocVersion ?? "not available — pandoc columns are empty")279        * LibreOffice: \(r.slowChecksRun ? (r.libreOfficeVersion ?? "available") : "not run (needs `ORIX_SLOW_TESTS=1` and `soffice`)")280        * textutil: \(r.textutilAvailable ? "available (macOS)" : "not available")281        * Microsoft Word: **not used**. Nothing in this report is a statement about how Word itself opens these files.282283        ## Cycle284285        Each document goes through Import → Export → Reimport → Export → Reimport. "Model equal" compares the whole document model286        (text, formatting, structure, notes, comments, revisions, assets, preserved parts) with generated identifiers ignored.287288        | Measure | Result |289        |---|---|290        | Documents | \(n("documents")) |291        | Imported without error | \(n("imported")) / \(n("documents")) |292        | Model equal after the first cycle (import vs reimport) | \(n("modelEqualAfterFirstCycle")) / \(n("documents")) |293        | Model equal after the second cycle | \(n("modelEqualAfterSecondCycle")) / \(n("documents")) |294        | Words in the corpus (body and notes, as imported) | \(n("words")) |295        | Text integrity after two cycles (word-weighted) | \(String(format: "%.2f %%", r.totals["textIntegrityAfterTwoCyclesPercent"] ?? 0)) |296        | Feature checks passed / failed (feature × document) | \(n("featurePass")) / \(n("featureFail")) |297        | Import warnings: info / approximated / dropped | \(n("importWarningsInfo")) / \(n("importWarningsApproximated")) / \(n("importWarningsDropped")) |298        | Export warnings (all severities) | \(n("exportWarnings")) |299300        ## Independent readers301302        | Measure | Result |303        |---|---|304        | pandoc reads our export | \(n("pandocReadsExport")) / \(r.documents.filter { $0.pandocReadsExport != nil }.count) |305        | Words pandoc reads in the original that it also reads, in order, in our export (mean) | \(String(format: "%.1f %%", r.totals["pandocTextIntegrityOfExportPercentMean"] ?? 0)) |306        | Words in pandoc's AST of the original (text and code; math and image alternative text excluded) that our importer also has, in order (mean) | \(String(format: "%.1f %%", r.totals["importAgreementWithPandocPercentMean"] ?? 0)) |307        | textutil reads our export | \(n("textutilReadsExport")) / \(r.documents.filter { $0.textutilReadsExport != nil }.count) |308        | Words textutil reads in the original that it also reads in our export (mean) | \(String(format: "%.1f %%", r.totals["textutilTextIntegrityOfExportPercentMean"] ?? 0)) |309        | LibreOffice converts our export to PDF | \(r.slowChecksRun ? "\(n("libreOfficeConvertsExportToPDF")) / \(r.documents.filter { $0.libreOfficeConvertsExportToPDF != nil }.count)" : "not run") |310311        Agreement below 100 % is expected where the tools and ORIX Docs deliberately differ: pandoc and textutil print cached field312        results, generated contents (and their title) and bibliographies as text, while ORIX Docs imports them as fields and313        generated blocks and recomputes them; pandoc keeps the text of a tracked move at its origin while ORIX Docs imports it as314        a deletion; hand-authored fixtures contain stale cached field text on purpose.315316        ## Per document317318        | Document | Source | Words | Model = after cycle 1 / 2 | Text after 2 cycles | Import vs pandoc | pandoc reads export (text) | textutil reads export (text) | LibreOffice PDF | Import warnings (info / approx. / dropped) |319        |---|---|---:|---|---:|---:|---|---|---|---|320321        """322        for d in r.documents {323            s += "| `\(d.name)` | \(d.source) | \(d.wordsInDocument) | \(yn(d.modelEqualAfterFirstCycle)) / \(yn(d.modelEqualAfterSecondCycle)) | \(pct(d.textIntegrityAfterTwoCycles)) | \(pct(d.importAgreementWithPandoc)) | \(yn(d.pandocReadsExport)) (\(pct(d.pandocTextIntegrityOfExport))) | \(yn(d.textutilReadsExport)) (\(pct(d.textutilTextIntegrityOfExport))) | \(yn(d.libreOfficeConvertsExportToPDF)) | \(d.importWarnings["info"] ?? 0) / \(d.importWarnings["approximated"] ?? 0) / \(d.importWarnings["dropped"] ?? 0) |\n"324        }325        s += "\n## Per feature\n\nA feature passes for a document when the number of its instances is the same after the import, after the first cycle and after the second cycle.\n\n| Feature | Documents that contain it | Instances | Pass | Fail |\n|---|---:|---:|---:|---:|\n"326        for f in r.featureTotals.keys.sorted() {327            let t = r.featureTotals[f]!328            s += "| \(f) | \(t["documents"]!) | \(t["instances"]!) | \(t["pass"]!) | \(t["fail"]!) |\n"329        }330        s += "\n## Import warnings by document\n\nWarnings of severity *approximated* or *dropped* raised while importing (the hand-authored packages contain unsupported content on purpose).\n\n"331        for d in r.documents where !d.warningFeatures.isEmpty {332            s += "* `\(d.name)`: " + d.warningFeatures.joined(separator: "; ") + "\n"333        }334        s += "\n## Limits of this report\n\n* The corpus is small (\(n("documents")) documents) and mostly synthetic; it measures the engine on the features listed in `docs/compatibility/docx.md`, not on arbitrary real-world documents.\n* \"Model equal\" proves that ORIX Docs reads back what it writes. It does not prove that another application renders the file the same way.\n* Layout (line breaks, pagination, fonts) is not measured anywhere in this report.\n* Microsoft Word was not launched; no claim is made about Word.\n"335        return s336    }337}338
Files main
Menu

Theme