Repository Public
1// ORIX Docs2// File: Packages/ORIXDocsKit/Tests/DocDOCXTests/CompatibilityReportTests.swift3// Description: Runs the whole corpus through Import → Export → Reimport → Export → Reimport, measures what survives (per feature,4// text integrity, agreement with pandoc / textutil / LibreOffice) and writes the DOCX compatibility report5// (Markdown + JSON) when ORIX_DOCX_REPORT_DIR is set. Every number in the report is measured here.6//7// Author: Simon-Pierre Boucher8// Contact: contact@spboucher.ai9import Foundation10import Testing11import DocModel12import DocContainer13@testable import DocDOCX1415struct FeatureCensus: Codable, Equatable {16 var counts: [String: Int] = [:]1718 init(_ doc: DocumentModel) {19 func add(_ k: String, _ n: Int = 1) { counts[k, default: 0] += n }20 let headingStyles = Set(doc.styles.styles.filter { (doc.styles.styleProps($0.id).para.outlineLevel ?? 0) >= 1 }.map(\.id))21 func scan(_ p: Paragraph, body: Bool) {22 if body {23 add("paragraphs")24 if headingStyles.contains(p.style) || (p.props.outlineLevel ?? 0) >= 1 { add("headings") }25 if p.list != nil { add("list items") }26 }27 if p.markRevision != nil { add("tracked paragraph marks") }28 for r in p.runs {29 if r.revision?.kind == .insertion { add("tracked insertions") }30 if r.revision?.kind == .deletion { add("tracked deletions") }31 if r.formatRevision != nil { add("tracked format changes") }32 if r.props.link != nil { add("hyperlink runs") }33 if !r.comments.isEmpty { add("commented runs") }34 if r.props.bold == true { add("bold runs") }35 if r.props.italic == true { add("italic runs") }36 switch r.inline {37 case .image?: add("images")38 case .shape?: add("shapes")39 case .equation?: add("equations")40 case .note(let n)?: add(n.kind == .footnote ? "footnote references" : "endnote references")41 case .citation?: add("citations")42 case .crossRef?: add("cross-references")43 case .field?: add("fields")44 case .bookmark?: add("bookmarks")45 case nil: break46 }47 }48 }49 doc.forEachBodyParagraph { p, _ in scan(p, body: true) }50 for n in doc.notes.values { for p in n.paragraphs { scan(p, body: false) } }51 for s in doc.sectionSetups {52 for p in s.header.default + s.header.first + s.header.even { add("header paragraphs"); scan(p, body: false) }53 for p in s.footer.default + s.footer.first + s.footer.even { add("footer paragraphs"); scan(p, body: false) }54 }55 for b in doc.body {56 switch b {57 case .table(let t):58 add("tables")59 add("table rows", t.rows.count)60 for r in t.rows { for c in r.cells { add("table cells"); if c.colSpan > 1 || c.vMerge != .none { add("merged cells") } } }61 case .pageBreak: add("page and column breaks")62 case .sectionBreak: add("section breaks")63 case .toc: add("generated lists (TOC)")64 case .bibliography: add("bibliography blocks")65 case .paragraph: break66 }67 }68 add("notes", doc.notes.count)69 add("comments", doc.comments.count)70 add("comment replies", doc.comments.reduce(0) { $0 + $1.replies.count })71 add("styles", doc.styles.styles.count)72 add("list definitions", doc.lists.count)73 add("image assets", doc.assets.count)74 add("preserved parts", doc.preservedParts.count)75 add("bibliography entries", doc.bibliography.count)76 counts = counts.filter { $0.value > 0 }77 }78}7980struct DocumentReport: Codable {81 var name: String82 var source: String83 var bytes: Int84 var imported = false85 var importError: String?86 var importWarnings: [String: Int] = [:]87 var warningFeatures: [String] = []88 var exportWarnings: [String: Int] = [:]89 var census: [String: Int] = [:]90 var featuresKept: [String: Bool] = [:]91 var modelEqualAfterFirstCycle = false92 var modelEqualAfterSecondCycle = false93 var textIntegrityAfterTwoCycles: Double = 094 var wordsInDocument = 095 var importAgreementWithPandoc: Double?96 var pandocReadsExport: Bool?97 var pandocTextIntegrityOfExport: Double?98 var textutilReadsExport: Bool?99 var textutilTextIntegrityOfExport: Double?100 var libreOfficeConvertsExportToPDF: Bool?101}102103struct CompatibilityReport: Codable {104 var generated: String105 var pandocVersion: String?106 var libreOfficeVersion: String?107 var textutilAvailable: Bool108 var slowChecksRun: Bool109 var microsoftWordUsed = false110 var documents: [DocumentReport] = []111 var featureTotals: [String: [String: Int]] = [:]112 var totals: [String: Double] = [:]113}114115@Suite("DOCX corpus cycle and compatibility report", .serialized)116struct CompatibilityReportTests {117 func allWords(_ doc: DocumentModel) -> [String] {118 var words: [String] = []119 doc.forEachBodyParagraph { p, _ in words += wordSequence(p.visibleText) }120 let n = Numbering(doc)121 for id in n.footnoteOrder + n.endnoteOrder { for p in doc.notes[id]?.paragraphs ?? [] { words += wordSequence(p.visibleText) } }122 return words123 }124125 func severityCounts(_ w: [FormatWarning]) -> [String: Int] {126 var out: [String: Int] = [:]127 for x in w { out[x.severity.rawValue, default: 0] += x.count }128 return out129 }130131 func version(_ tool: String?, _ args: [String]) -> String? {132 guard let tool else { return nil }133 let r = Tool.run(tool, args, timeout: 60)134 return String(decoding: r.stdout, as: UTF8.self).split(separator: "\n").first.map(String.init)135 }136137 @Test func corpusCycle() throws {138 var sources: [(name: String, source: String, data: Data)] = []139 for name in Corpus.toolFiles { sources.append((name, name.hasPrefix("pandoc") ? "pandoc" : "LibreOffice", try Corpus.data(name))) }140 for (name, data) in try Hand.all() { sources.append((name, "hand-authored OOXML", data)) }141 var options = DocxExportOptions()142 options.citations = FixedFormatter()143 sources.append(("orix-rich.docx", "ORIX Docs model", try DocxExporter.export(richDocument(), options: options).data))144145 if let dump = ProcessInfo.processInfo.environment["ORIX_DOCX_DUMP_DIR"] {146 try FileManager.default.createDirectory(atPath: dump, withIntermediateDirectories: true)147 for s in sources where s.source != "pandoc" && s.source != "LibreOffice" { try s.data.write(to: URL(fileURLWithPath: dump).appendingPathComponent(s.name)) }148 }149150 let slow = Tool.slowTestsEnabled && Tool.soffice != nil151 var report = CompatibilityReport(generated: ISO8601DateFormatter().string(from: Date()), pandocVersion: version(Tool.pandoc, ["--version"]),152 libreOfficeVersion: slow ? version(Tool.soffice, ["--version"]) : nil, textutilAvailable: Tool.textutil != nil, slowChecksRun: slow)153 if Tool.pandoc == nil { recordSkip("pandoc missing: pandoc columns of the compatibility report are empty") }154 if !slow { recordSkip("LibreOffice PDF conversion not measured in the compatibility report (needs ORIX_SLOW_TESTS=1 and soffice)") }155156 for s in sources {157 var d = DocumentReport(name: s.name, source: s.source, bytes: s.data.count)158 let a: ImportResult159 do { a = try DocxImporter.importDocument(s.data) } catch {160 d.importError = "\(error)"161 report.documents.append(d)162 Issue.record("\(s.name) cannot be imported: \(error)")163 continue164 }165 d.imported = true166 d.importWarnings = severityCounts(a.warnings)167 d.warningFeatures = Array(Set(a.warnings.filter { $0.severity != .info }.map { "\($0.severity.rawValue): \($0.feature)" })).sorted()168 let censusA = FeatureCensus(a.document)169 d.census = censusA.counts170171 let export1 = try DocxExporter.export(a.document, options: options)172 d.exportWarnings = severityCounts(export1.warnings)173 let b = try DocxImporter.importDocument(export1.data).document174 let export2 = try DocxExporter.export(b, options: options)175 let c = try DocxImporter.importDocument(export2.data).document176 let censusB = FeatureCensus(b), censusC = FeatureCensus(c)177 for (feature, n) in censusA.counts { d.featuresKept[feature] = censusB.counts[feature] == n && censusC.counts[feature] == n }178 d.modelEqualAfterFirstCycle = normalized(a.document) == normalized(b)179 d.modelEqualAfterSecondCycle = normalized(b) == normalized(c)180 let words = allWords(a.document)181 d.wordsInDocument = words.count182 d.textIntegrityAfterTwoCycles = textIntegrity(reference: words, candidate: allWords(c))183184 if Tool.pandoc != nil {185 if let astWords = Tool.pandocWords(s.data) { d.importAgreementWithPandoc = textIntegrity(reference: astWords, candidate: words) }186 if let original = Tool.pandocText(s.data) {187 if let ours = Tool.pandocText(export1.data) {188 d.pandocReadsExport = true189 d.pandocTextIntegrityOfExport = textIntegrity(reference: wordSequence(original), candidate: wordSequence(ours))190 } else { d.pandocReadsExport = false }191 }192 }193 if Tool.textutil != nil, let original = Tool.textutilText(s.data) {194 if let ours = Tool.textutilText(export1.data) {195 d.textutilReadsExport = true196 d.textutilTextIntegrityOfExport = textIntegrity(reference: wordSequence(original), candidate: wordSequence(ours))197 } else { d.textutilReadsExport = false }198 }199 if slow { d.libreOfficeConvertsExportToPDF = Tool.sofficeConvert(export1.data, to: "pdf").map { $0.prefix(5) == Data("%PDF-".utf8) } ?? false }200 report.documents.append(d)201202 // The cycle itself is a hard requirement, not just a measurement.203 expectSameModel(a.document, b, "\(s.name), first cycle")204 expectSameModel(b, c, "\(s.name), second cycle")205 #expect(d.textIntegrityAfterTwoCycles == 1.0, "\(s.name): text integrity after two cycles")206 #expect(d.featuresKept.values.allSatisfy { $0 }, "\(s.name): features changed count: \(d.featuresKept.filter { !$0.value }.keys.sorted())")207 if let ok = d.pandocReadsExport { #expect(ok, "pandoc cannot read the export of \(s.name)") }208 if let ok = d.textutilReadsExport { #expect(ok, "textutil cannot read the export of \(s.name)") }209 if let ok = d.libreOfficeConvertsExportToPDF { #expect(ok, "LibreOffice cannot convert the export of \(s.name)") }210 }211212 // Totals.213 var features: [String: [String: Int]] = [:]214 for d in report.documents {215 for (f, kept) in d.featuresKept {216 features[f, default: ["documents": 0, "pass": 0, "fail": 0, "instances": 0]]["documents"]! += 1217 features[f]![kept ? "pass" : "fail"]! += 1218 features[f]!["instances"]! += d.census[f] ?? 0219 }220 }221 report.featureTotals = features222 let docs = report.documents223 func mean(_ xs: [Double]) -> Double { xs.isEmpty ? 0 : xs.reduce(0, +) / Double(xs.count) }224 func weighted(_ pick: (DocumentReport) -> Double?) -> Double {225 let rows = docs.compactMap { d in pick(d).map { ($0, Double(d.wordsInDocument)) } }226 let w = rows.reduce(0) { $0 + $1.1 }227 return w == 0 ? 0 : rows.reduce(0) { $0 + $1.0 * $1.1 } / w228 }229 report.totals = [230 "documents": Double(docs.count),231 "imported": Double(docs.filter(\.imported).count),232 "modelEqualAfterFirstCycle": Double(docs.filter(\.modelEqualAfterFirstCycle).count),233 "modelEqualAfterSecondCycle": Double(docs.filter(\.modelEqualAfterSecondCycle).count),234 "words": Double(docs.reduce(0) { $0 + $1.wordsInDocument }),235 "textIntegrityAfterTwoCyclesPercent": weighted { $0.textIntegrityAfterTwoCycles } * 100,236 "importAgreementWithPandocPercentMean": mean(docs.compactMap(\.importAgreementWithPandoc)) * 100,237 "pandocReadsExport": Double(docs.filter { $0.pandocReadsExport == true }.count),238 "pandocTextIntegrityOfExportPercentMean": mean(docs.compactMap(\.pandocTextIntegrityOfExport)) * 100,239 "textutilReadsExport": Double(docs.filter { $0.textutilReadsExport == true }.count),240 "textutilTextIntegrityOfExportPercentMean": mean(docs.compactMap(\.textutilTextIntegrityOfExport)) * 100,241 "libreOfficeConvertsExportToPDF": Double(docs.filter { $0.libreOfficeConvertsExportToPDF == true }.count),242 "importWarningsInfo": Double(docs.reduce(0) { $0 + ($1.importWarnings["info"] ?? 0) }),243 "importWarningsApproximated": Double(docs.reduce(0) { $0 + ($1.importWarnings["approximated"] ?? 0) }),244 "importWarningsDropped": Double(docs.reduce(0) { $0 + ($1.importWarnings["dropped"] ?? 0) }),245 "exportWarnings": Double(docs.reduce(0) { $0 + $1.exportWarnings.values.reduce(0, +) }),246 "featurePass": Double(features.values.reduce(0) { $0 + $1["pass"]! }),247 "featureFail": Double(features.values.reduce(0) { $0 + $1["fail"]! }),248 ]249 #expect(report.totals["imported"] == report.totals["documents"])250251 guard let dir = ProcessInfo.processInfo.environment["ORIX_DOCX_REPORT_DIR"] else {252 recordSkip("ORIX_DOCX_REPORT_DIR not set: compatibility report measured but not written")253 return254 }255 let enc = JSONEncoder()256 enc.outputFormatting = [.prettyPrinted, .sortedKeys]257 try FileManager.default.createDirectory(atPath: dir, withIntermediateDirectories: true)258 try enc.encode(report).write(to: URL(fileURLWithPath: dir).appendingPathComponent("docx-compatibility.json"))259 try markdown(report).write(to: URL(fileURLWithPath: dir).appendingPathComponent("docx-compatibility.md"), atomically: true, encoding: .utf8)260 }261262 func markdown(_ r: CompatibilityReport) -> String {263 func pct(_ v: Double?) -> String { v.map { String(format: "%.1f %%", $0 * 100) } ?? "n/a" }264 func yn(_ v: Bool?) -> String { v.map { $0 ? "yes" : "NO" } ?? "n/a" }265 func n(_ key: String) -> Int { Int(r.totals[key] ?? 0) }266 var s = """267 <!--268 ORIX Docs269 File: docs/reports/docx-compatibility.md270 Description: Measured DOCX compatibility of the ORIX Docs engine on the reference corpus. Generated file — do not edit; run Scripts/docx-compat.sh.271 Author: Simon-Pierre Boucher272 Contact: contact@spboucher.ai273 -->274 # DOCX compatibility report (measured)275276 Generated \(r.generated) by `Scripts/docx-compat.sh` (test `CompatibilityReportTests.corpusCycle`). Every number below was measured during that run.277278 * pandoc: \(r.pandocVersion ?? "not available — pandoc columns are empty")279 * LibreOffice: \(r.slowChecksRun ? (r.libreOfficeVersion ?? "available") : "not run (needs `ORIX_SLOW_TESTS=1` and `soffice`)")280 * textutil: \(r.textutilAvailable ? "available (macOS)" : "not available")281 * Microsoft Word: **not used**. Nothing in this report is a statement about how Word itself opens these files.282283 ## Cycle284285 Each document goes through Import → Export → Reimport → Export → Reimport. "Model equal" compares the whole document model286 (text, formatting, structure, notes, comments, revisions, assets, preserved parts) with generated identifiers ignored.287288 | Measure | Result |289 |---|---|290 | Documents | \(n("documents")) |291 | Imported without error | \(n("imported")) / \(n("documents")) |292 | Model equal after the first cycle (import vs reimport) | \(n("modelEqualAfterFirstCycle")) / \(n("documents")) |293 | Model equal after the second cycle | \(n("modelEqualAfterSecondCycle")) / \(n("documents")) |294 | Words in the corpus (body and notes, as imported) | \(n("words")) |295 | Text integrity after two cycles (word-weighted) | \(String(format: "%.2f %%", r.totals["textIntegrityAfterTwoCyclesPercent"] ?? 0)) |296 | Feature checks passed / failed (feature × document) | \(n("featurePass")) / \(n("featureFail")) |297 | Import warnings: info / approximated / dropped | \(n("importWarningsInfo")) / \(n("importWarningsApproximated")) / \(n("importWarningsDropped")) |298 | Export warnings (all severities) | \(n("exportWarnings")) |299300 ## Independent readers301302 | Measure | Result |303 |---|---|304 | pandoc reads our export | \(n("pandocReadsExport")) / \(r.documents.filter { $0.pandocReadsExport != nil }.count) |305 | Words pandoc reads in the original that it also reads, in order, in our export (mean) | \(String(format: "%.1f %%", r.totals["pandocTextIntegrityOfExportPercentMean"] ?? 0)) |306 | Words in pandoc's AST of the original (text and code; math and image alternative text excluded) that our importer also has, in order (mean) | \(String(format: "%.1f %%", r.totals["importAgreementWithPandocPercentMean"] ?? 0)) |307 | textutil reads our export | \(n("textutilReadsExport")) / \(r.documents.filter { $0.textutilReadsExport != nil }.count) |308 | Words textutil reads in the original that it also reads in our export (mean) | \(String(format: "%.1f %%", r.totals["textutilTextIntegrityOfExportPercentMean"] ?? 0)) |309 | LibreOffice converts our export to PDF | \(r.slowChecksRun ? "\(n("libreOfficeConvertsExportToPDF")) / \(r.documents.filter { $0.libreOfficeConvertsExportToPDF != nil }.count)" : "not run") |310311 Agreement below 100 % is expected where the tools and ORIX Docs deliberately differ: pandoc and textutil print cached field312 results, generated contents (and their title) and bibliographies as text, while ORIX Docs imports them as fields and313 generated blocks and recomputes them; pandoc keeps the text of a tracked move at its origin while ORIX Docs imports it as314 a deletion; hand-authored fixtures contain stale cached field text on purpose.315316 ## Per document317318 | Document | Source | Words | Model = after cycle 1 / 2 | Text after 2 cycles | Import vs pandoc | pandoc reads export (text) | textutil reads export (text) | LibreOffice PDF | Import warnings (info / approx. / dropped) |319 |---|---|---:|---|---:|---:|---|---|---|---|320321 """322 for d in r.documents {323 s += "| `\(d.name)` | \(d.source) | \(d.wordsInDocument) | \(yn(d.modelEqualAfterFirstCycle)) / \(yn(d.modelEqualAfterSecondCycle)) | \(pct(d.textIntegrityAfterTwoCycles)) | \(pct(d.importAgreementWithPandoc)) | \(yn(d.pandocReadsExport)) (\(pct(d.pandocTextIntegrityOfExport))) | \(yn(d.textutilReadsExport)) (\(pct(d.textutilTextIntegrityOfExport))) | \(yn(d.libreOfficeConvertsExportToPDF)) | \(d.importWarnings["info"] ?? 0) / \(d.importWarnings["approximated"] ?? 0) / \(d.importWarnings["dropped"] ?? 0) |\n"324 }325 s += "\n## Per feature\n\nA feature passes for a document when the number of its instances is the same after the import, after the first cycle and after the second cycle.\n\n| Feature | Documents that contain it | Instances | Pass | Fail |\n|---|---:|---:|---:|---:|\n"326 for f in r.featureTotals.keys.sorted() {327 let t = r.featureTotals[f]!328 s += "| \(f) | \(t["documents"]!) | \(t["instances"]!) | \(t["pass"]!) | \(t["fail"]!) |\n"329 }330 s += "\n## Import warnings by document\n\nWarnings of severity *approximated* or *dropped* raised while importing (the hand-authored packages contain unsupported content on purpose).\n\n"331 for d in r.documents where !d.warningFeatures.isEmpty {332 s += "* `\(d.name)`: " + d.warningFeatures.joined(separator: "; ") + "\n"333 }334 s += "\n## Limits of this report\n\n* The corpus is small (\(n("documents")) documents) and mostly synthetic; it measures the engine on the features listed in `docs/compatibility/docx.md`, not on arbitrary real-world documents.\n* \"Model equal\" proves that ORIX Docs reads back what it writes. It does not prove that another application renders the file the same way.\n* Layout (line breaks, pagination, fonts) is not measured anywhere in this report.\n* Microsoft Word was not launched; no claim is made about Word.\n"335 return s336 }337}338