Repository Public
1// ORIX Docs2// File: Packages/ORIXDocsKit/Tests/DocDOCXTests/TestSupport.swift3// Description: Shared helpers of the DOCX tests: corpus access, hand-built OOXML packages, external tool runner,4// identifier-insensitive model comparison and text extraction.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import Testing10import DocModel11import DocContainer12@testable import DocDOCX1314// MARK: - Corpus1516enum Corpus {17 static var directory: URL { Bundle.module.resourceURL!.appendingPathComponent("Corpus") }18 static func data(_ name: String) throws -> Data { try Data(contentsOf: directory.appendingPathComponent(name)) }19 static var toolFiles: [String] {20 ((try? FileManager.default.contentsOfDirectory(atPath: directory.path)) ?? []).filter { $0.hasSuffix(".docx") }.sorted()21 }22}2324// MARK: - External tools2526enum Tool {27 static func path(_ name: String) -> String? {28 for dir in ["/opt/homebrew/bin", "/usr/local/bin", "/usr/bin", "/bin"] {29 let p = dir + "/" + name30 if FileManager.default.isExecutableFile(atPath: p) { return p }31 }32 return nil33 }34 static var pandoc: String? { path("pandoc") }35 static var soffice: String? { path("soffice") }36 static var textutil: String? { path("textutil") }37 static var slowTestsEnabled: Bool { ProcessInfo.processInfo.environment["ORIX_SLOW_TESTS"] == "1" }3839 struct Output { var status: Int32; var stdout: Data; var stderr: String }4041 static func run(_ executable: String, _ arguments: [String], cwd: URL? = nil, timeout: TimeInterval = 180) -> Output {42 let p = Process()43 p.executableURL = URL(fileURLWithPath: executable)44 p.arguments = arguments45 if let cwd { p.currentDirectoryURL = cwd }46 let out = Pipe(), err = Pipe()47 p.standardOutput = out48 p.standardError = err49 p.standardInput = FileHandle.nullDevice50 do { try p.run() } catch { return Output(status: -1, stdout: Data(), stderr: "\(error)") }51 let killer = DispatchWorkItem { if p.isRunning { p.terminate() } }52 DispatchQueue.global().asyncAfter(deadline: .now() + timeout, execute: killer)53 let data = out.fileHandleForReading.readDataToEndOfFile()54 let errData = err.fileHandleForReading.readDataToEndOfFile()55 p.waitUntilExit()56 killer.cancel()57 return Output(status: p.terminationStatus, stdout: data, stderr: String(decoding: errData, as: UTF8.self))58 }5960 static func tempDirectory() -> URL {61 let url = FileManager.default.temporaryDirectory.appendingPathComponent("orix-docx-tests-" + UUID().uuidString)62 try? FileManager.default.createDirectory(at: url, withIntermediateDirectories: true)63 return url64 }6566 /// pandoc's view of a DOCX: plain text and the JSON AST.67 static func pandocText(_ docx: Data, to format: String = "plain", extra: [String] = []) -> String? {68 guard let pandoc else { return nil }69 let dir = tempDirectory()70 defer { try? FileManager.default.removeItem(at: dir) }71 let file = dir.appendingPathComponent("in.docx")72 try? docx.write(to: file)73 let r = run(pandoc, ["-f", "docx", "-t", format, "--wrap=none"] + extra + [file.path])74 guard r.status == 0 else { return nil }75 return String(decoding: r.stdout, as: UTF8.self)76 }7778 /// The words pandoc reads in a DOCX, from its AST: text and code only (no list markers, no math rendering, no table rules,79 /// no image alternative text), body first, then the notes in order of reference.80 static func pandocWords(_ docx: Data) -> [String]? {81 guard let json = pandocText(docx, to: "json"), let ast = try? JSONSerialization.jsonObject(with: Data(json.utf8)) as? [String: Any] else { return nil }82 var body = "", notes = ""83 let inlineTypes: Set<String> = ["Str", "Emph", "Underline", "Strong", "Strikeout", "Superscript", "Subscript", "SmallCaps", "Quoted", "Cite", "Code", "Space",84 "SoftBreak", "LineBreak", "Math", "RawInline", "Link", "Image", "Note", "Span"]85 func emit(_ text: String, _ inNote: Bool) { if inNote { notes += text } else { body += text } }86 func walk(_ any: Any, inNote: Bool) {87 if let d = any as? [String: Any] {88 guard let t = d["t"] as? String else {89 if let blocks = d["blocks"] { walk(blocks, inNote: inNote) }90 return91 }92 switch t {93 case "Str": emit(d["c"] as? String ?? "", inNote)94 case "Code", "CodeBlock": emit(((d["c"] as? [Any])?.last as? String ?? "") + (t == "CodeBlock" ? "\n" : ""), inNote)95 case "Space", "SoftBreak", "LineBreak": emit(" ", inNote)96 case "Math", "Image", "RawInline", "RawBlock": break97 case "Quoted":98 emit("\"", inNote)99 if let c = d["c"] { walk(c, inNote: inNote) }100 emit("\"", inNote)101 case "Note":102 if let c = d["c"] { walk(c, inNote: true) }103 default:104 if let c = d["c"] { walk(c, inNote: inNote) }105 if !inlineTypes.contains(t) { emit("\n", inNote) }106 }107 } else if let a = any as? [Any] {108 for v in a { walk(v, inNote: inNote) }109 }110 }111 walk(ast, inNote: false)112 return wordSequence(body) + wordSequence(notes)113 }114115 static func textutilText(_ docx: Data) -> String? {116 guard let textutil else { return nil }117 let dir = tempDirectory()118 defer { try? FileManager.default.removeItem(at: dir) }119 let file = dir.appendingPathComponent("in.docx")120 try? docx.write(to: file)121 let r = run(textutil, ["-convert", "txt", "-stdout", file.path])122 guard r.status == 0 else { return nil }123 return String(decoding: r.stdout, as: UTF8.self)124 }125126 /// Converts with LibreOffice headless (private profile, no window). Returns the produced file's bytes.127 static func sofficeConvert(_ docx: Data, to format: String) -> Data? {128 guard let soffice else { return nil }129 let dir = tempDirectory()130 defer { try? FileManager.default.removeItem(at: dir) }131 let file = dir.appendingPathComponent("in.docx")132 try? docx.write(to: file)133 let profile = dir.appendingPathComponent("profile")134 let r = run(soffice, ["-env:UserInstallation=file://\(profile.path)", "--headless", "--convert-to", format, "--outdir", dir.path, file.path], timeout: 240)135 guard r.status == 0 else { return nil }136 let ext = format.split(separator: ":").first.map(String.init) ?? format137 return try? Data(contentsOf: dir.appendingPathComponent("in." + ext))138 }139}140141/// Notes about checks that could not run (missing tool, slow tests disabled). Printed so a skipped check is never silent.142func recordSkip(_ what: String) {143 print("ORIX-DOCX-SKIP: \(what)")144}145146// MARK: - Hand-built packages147148enum Pkg {149 static let ns = "xmlns:w=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\" xmlns:r=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships\" xmlns:m=\"http://schemas.openxmlformats.org/officeDocument/2006/math\" xmlns:wp=\"http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing\" xmlns:a=\"http://schemas.openxmlformats.org/drawingml/2006/main\" xmlns:pic=\"http://schemas.openxmlformats.org/drawingml/2006/picture\" xmlns:mc=\"http://schemas.openxmlformats.org/markup-compatibility/2006\" xmlns:w14=\"http://schemas.microsoft.com/office/word/2010/wordml\" xmlns:w15=\"http://schemas.microsoft.com/office/word/2012/wordml\" xmlns:wps=\"http://schemas.microsoft.com/office/word/2010/wordprocessingShape\" xmlns:v=\"urn:schemas-microsoft-com:vml\" xmlns:o=\"urn:schemas-microsoft-com:office:office\""150 static let head = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>"151152 /// A minimal package around a body. `parts` adds or replaces entries; `documentRels` adds relationships of the main part153 /// (id, type suffix, target, external).154 static func make(body: String, parts: [String: String] = [:], binary: [String: Data] = [:],155 documentRels: [(String, String, String, Bool)] = [], mainType: String = "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml",156 overrides: [String: String] = [:]) throws -> Data {157 var files: [String: Data] = [:]158 var types = head + "<Types xmlns=\"http://schemas.openxmlformats.org/package/2006/content-types\"><Default Extension=\"rels\" ContentType=\"application/vnd.openxmlformats-package.relationships+xml\"/><Default Extension=\"xml\" ContentType=\"application/xml\"/><Default Extension=\"png\" ContentType=\"image/png\"/><Default Extension=\"bin\" ContentType=\"application/vnd.ms-office.vbaProject\"/><Override PartName=\"/word/document.xml\" ContentType=\"\(mainType)\"/>"159 for (k, v) in overrides.sorted(by: { $0.key < $1.key }) { types += "<Override PartName=\"/\(k)\" ContentType=\"\(v)\"/>" }160 types += "</Types>"161 files["[Content_Types].xml"] = Data(types.utf8)162 files["_rels/.rels"] = Data((head + "<Relationships xmlns=\"http://schemas.openxmlformats.org/package/2006/relationships\"><Relationship Id=\"rId1\" Type=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument\" Target=\"word/document.xml\"/></Relationships>").utf8)163 files["word/document.xml"] = Data((head + "<w:document \(ns)><w:body>" + body + "</w:body></w:document>").utf8)164 var rels = head + "<Relationships xmlns=\"http://schemas.openxmlformats.org/package/2006/relationships\">"165 let base = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/"166 for (id, type, target, external) in documentRels {167 rels += "<Relationship Id=\"\(id)\" Type=\"\(type.contains("/") ? type : base + type)\" Target=\"\(target)\"\(external ? " TargetMode=\"External\"" : "")/>"168 }169 rels += "</Relationships>"170 files["word/_rels/document.xml.rels"] = Data(rels.utf8)171 for (k, v) in parts { files[k] = Data(v.utf8) }172 for (k, v) in binary { files[k] = v }173 var zip = ZipWriter()174 try zip.add("[Content_Types].xml", files["[Content_Types].xml"]!)175 for k in files.keys.sorted() where k != "[Content_Types].xml" { try zip.add(k, files[k]!) }176 return zip.finalize()177 }178179 static func p(_ text: String, style: String? = nil) -> String {180 "<w:p>" + (style.map { "<w:pPr><w:pStyle w:val=\"\($0)\"/></w:pPr>" } ?? "") + "<w:r><w:t xml:space=\"preserve\">\(text)</w:t></w:r></w:p>"181 }182183 /// A tiny valid PNG (1 × 1, opaque).184 static let png = Data(base64Encoded: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==")!185}186187/// Unzips an exported package into name → XML string (text parts only).188func unpack(_ data: Data) throws -> [String: String] {189 let zip = try ZipReader(data: data)190 var out: [String: String] = [:]191 for n in zip.names { if let d = try zip.data(for: n), n.hasSuffix(".xml") || n.hasSuffix(".rels") { out[n] = String(decoding: d, as: UTF8.self) } }192 return out193}194195// MARK: - Model comparison196197/// Rewrites every generated identifier (paragraphs, tables, lists, notes, comments, revisions, assets) to a value that depends only198/// on the order of appearance, so two models that differ only by identifiers compare equal.199func normalized(_ input: DocumentModel) -> DocumentModel {200 var d = input201 var paraIDs: [String: String] = [:], noteIDs: [String: String] = [:], revIDs: [String: String] = [:], fmtIDs: [String: String] = [:]202 var listIDs: [String: String] = [:], commentIDs: [String: String] = [:], assetIDs: [String: String] = [:]203 for (i, l) in d.lists.enumerated() { listIDs[l.id] = "l\(i + 1)"; d.lists[i].id = "l\(i + 1)" }204 for i in d.comments.indices {205 commentIDs[d.comments[i].id] = "c\(i + 1)"206 d.comments[i].id = "c\(i + 1)"207 for k in d.comments[i].replies.indices { d.comments[i].replies[k].id = "c\(i + 1)r\(k + 1)" }208 }209 for (id, a) in d.assets { assetIDs[id] = "a" + String(CRC32.checksum(a.data), radix: 16) }210 d.assets = Dictionary(d.assets.map { (assetIDs[$0.key]!, Asset(id: assetIDs[$0.key]!, mimeType: $0.value.mimeType, data: $0.value.data)) }, uniquingKeysWith: { a, _ in a })211212 var noteOrder: [String] = []213 func pass1(_ p: Paragraph) {214 if paraIDs[p.id] == nil { paraIDs[p.id] = "p\(paraIDs.count + 1)" }215 for r in p.runs { if case .note(let n)? = r.inline, noteIDs[n.id] == nil { noteIDs[n.id] = "n\(noteIDs.count + 1)"; noteOrder.append(n.id) } }216 }217 func rev(_ r: Revision?) -> Revision? {218 guard var r else { return nil }219 if revIDs[r.id] == nil { revIDs[r.id] = "rev\(revIDs.count + 1)" }220 r.id = revIDs[r.id]!221 return r222 }223 func pass2(_ p: inout Paragraph) {224 p.id = paraIDs[p.id] ?? p.id225 if let l = p.list { p.list = ListItem(listID: listIDs[l.listID] ?? l.listID, level: l.level) }226 p.markRevision = rev(p.markRevision)227 for i in p.runs.indices {228 p.runs[i].revision = rev(p.runs[i].revision)229 if var f = p.runs[i].formatRevision {230 if fmtIDs[f.id] == nil { fmtIDs[f.id] = "fmt\(fmtIDs.count + 1)" }231 f.id = fmtIDs[f.id]!232 p.runs[i].formatRevision = f233 }234 p.runs[i].comments = p.runs[i].comments.map { commentIDs[$0] ?? $0 }.sorted()235 switch p.runs[i].inline {236 case .note(var n)?: n.id = noteIDs[n.id] ?? n.id; p.runs[i].inline = .note(n)237 case .crossRef(var c)?: c.target = paraIDs[c.target] ?? c.target; p.runs[i].inline = .crossRef(c)238 case .image(var im)?: im.asset = assetIDs[im.asset] ?? im.asset; p.runs[i].inline = .image(im)239 default: break240 }241 }242 }243 func eachHF(_ s: inout PageSetup, _ f: (inout Paragraph) -> Void) {244 for i in s.header.default.indices { f(&s.header.default[i]) }245 for i in s.header.first.indices { f(&s.header.first[i]) }246 for i in s.header.even.indices { f(&s.header.even[i]) }247 for i in s.footer.default.indices { f(&s.footer.default[i]) }248 for i in s.footer.first.indices { f(&s.footer.first[i]) }249 for i in s.footer.even.indices { f(&s.footer.even[i]) }250 }251 func walk(_ f: (inout Paragraph) -> Void) {252 for bi in d.body.indices {253 switch d.body[bi] {254 case .paragraph(var p): f(&p); d.body[bi] = .paragraph(p)255 case .table(var t):256 t.id = "t"257 for ri in t.rows.indices {258 t.rows[ri].id = "r"259 for ci in t.rows[ri].cells.indices {260 t.rows[ri].cells[ci].id = "tc"261 for pi in t.rows[ri].cells[ci].paragraphs.indices { f(&t.rows[ri].cells[ci].paragraphs[pi]) }262 }263 }264 d.body[bi] = .table(t)265 case .pageBreak(var b): b.id = "b"; d.body[bi] = .pageBreak(b)266 case .sectionBreak(var s): s.id = "s"; eachHF(&s.setup, f); d.body[bi] = .sectionBreak(s)267 case .toc(var t): t.id = "toc"; d.body[bi] = .toc(t)268 case .bibliography(var b): b.id = "bib"; d.body[bi] = .bibliography(b)269 }270 }271 eachHF(&d.pageSetup, f)272 }273 walk { p in pass1(p) }274 // Notes in order of first reference, then the unreferenced ones by content.275 var k = 0276 while k < noteOrder.count {277 for p in d.notes[noteOrder[k]]?.paragraphs ?? [] { pass1(p) }278 k += 1279 }280 for id in d.notes.keys.sorted() where noteIDs[id] == nil { noteIDs[id] = "orphan-" + id; for p in d.notes[id]!.paragraphs { pass1(p) } }281 walk { p in pass2(&p) }282 var notes: [String: Note] = [:]283 for (id, var n) in d.notes {284 n.id = noteIDs[id]!285 for i in n.paragraphs.indices { pass2(&n.paragraphs[i]) }286 notes[n.id] = n287 }288 d.notes = notes289 return d290}291292/// A readable list of the differences between two models (first lines that differ in their pretty-printed JSON, plus assets).293func modelDifferences(_ a: DocumentModel, _ b: DocumentModel, limit: Int = 12) -> [String] {294 let enc = JSONEncoder()295 enc.outputFormatting = [.prettyPrinted, .sortedKeys]296 enc.dateEncodingStrategy = .iso8601297 let la = (try? enc.encode(a)).map { String(decoding: $0, as: UTF8.self).components(separatedBy: "\n") } ?? []298 let lb = (try? enc.encode(b)).map { String(decoding: $0, as: UTF8.self).components(separatedBy: "\n") } ?? []299 var out: [String] = []300 if la != lb {301 let diff = lb.difference(from: la)302 for change in diff {303 if out.count >= limit { break }304 switch change {305 case .insert(let o, let e, _): out.append("+\(o): \(e.trimmingCharacters(in: .whitespaces))")306 case .remove(let o, let e, _): out.append("-\(o): \(e.trimmingCharacters(in: .whitespaces))")307 }308 }309 }310 if out.isEmpty && a != b {311 // The JSON is identical: the difference is in something the encoding rounds (dates) or does not carry.312 var da = "", db = ""313 dump(a, to: &da); dump(b, to: &db)314 let xa = da.components(separatedBy: "\n"), xb = db.components(separatedBy: "\n")315 for (i, (l, r)) in zip(xa, xb).enumerated() where l != r { out.append("dump line \(i): \(l.trimmingCharacters(in: .whitespaces)) | \(r.trimmingCharacters(in: .whitespaces))"); if out.count >= limit { break } }316 }317 if a.assets != b.assets { out.append("assets differ: \(a.assets.keys.sorted()) vs \(b.assets.keys.sorted())") }318 if a.preservedParts != b.preservedParts { out.append("preserved parts differ: \(a.preservedParts.keys.sorted()) vs \(b.preservedParts.keys.sorted())") }319 return out320}321322/// Export, reimport and compare (identifier-insensitive). Returns the reimported model.323@discardableResult324func assertRoundTrip(_ doc: DocumentModel, options: DocxExportOptions = .init(), sourceLocation: SourceLocation = #_sourceLocation) throws -> DocumentModel {325 let exported = try DocxExporter.export(doc, options: options)326 let back = try DocxImporter.importDocument(exported.data).document327 let a = normalized(doc), b = normalized(back)328 if a != b {329 let details: String = modelDifferences(a, b).joined(separator: "\n")330 Issue.record(Comment(rawValue: "round trip changed the model:\n" + details), sourceLocation: sourceLocation)331 }332 return back333}334335// MARK: - Text extraction336337extension DocumentModel {338 /// Body text, one line per paragraph, table cells included, deleted text excluded, inline objects removed.339 var bodyLines: [String] {340 var out: [String] = []341 forEachBodyParagraph { p, _ in out.append(p.visibleText) }342 return out343 }344 var bodyParagraphs: [Paragraph] {345 var out: [Paragraph] = []346 forEachBodyParagraph { p, _ in out.append(p) }347 return out348 }349 func firstParagraph(containing text: String) -> Paragraph? { bodyParagraphs.first { $0.visibleText.contains(text) } }350 var tables: [Table] { body.compactMap(\.table) }351 var inlines: [Inline] { bodyParagraphs.flatMap { $0.runs.compactMap(\.inline) } }352}353354/// Letters and digits only: the comparison used for text integrity across tools that disagree on whitespace and punctuation glyphs.355func wordSequence(_ s: String) -> [String] {356 s.precomposedStringWithCanonicalMapping.split(whereSeparator: { !($0.isLetter || $0.isNumber) }).map { $0.lowercased() }357}358359/// Share of the words of `reference` that appear in `candidate` in the same order (longest common subsequence / reference length).360func textIntegrity(reference: [String], candidate: [String]) -> Double {361 guard !reference.isEmpty else { return candidate.isEmpty ? 1 : 0 }362 let removed = candidate.difference(from: reference).reduce(0) { n, c in if case .remove = c { return n + 1 } else { return n } }363 return Double(reference.count - removed) / Double(reference.count)364}365366/// A model whose values all survive DOCX exactly (half-point sizes, twentieths of a point), used by the round-trip tests.367func representableStyles() -> StyleSheet {368 var s: [Style] = []369 s.append(Style(id: StyleID.normal, name: "Body", para: ParaProps(spaceAfter: 8, lineSpacing: .multiple(1.25))))370 for lvl in 1...3 {371 s.append(Style(id: StyleID.heading(lvl), name: "Heading \(lvl)", basedOn: StyleID.normal, next: StyleID.normal,372 char: CharProps(fontFamily: "Helvetica Neue", fontSize: Double(20 - 2 * lvl), bold: true),373 para: ParaProps(spaceBefore: 16, spaceAfter: 6, keepWithNext: true, outlineLevel: lvl)))374 }375 s.append(Style(id: StyleID.caption, name: "Caption", basedOn: StyleID.normal, char: CharProps(fontSize: 10), para: ParaProps(alignment: .center)))376 s.append(Style(id: StyleID.footnoteText, name: "Footnote Text", basedOn: StyleID.normal, char: CharProps(fontSize: 9)))377 s.append(Style(id: StyleID.header, name: "Header", basedOn: StyleID.normal, para: ParaProps(tabs: [TabStop(position: 234, kind: .center), TabStop(position: 468, kind: .right)])))378 s.append(Style(id: StyleID.footer, name: "Footer", basedOn: StyleID.header))379 s.append(Style(id: StyleID.tableText, name: "Table Text", basedOn: StyleID.normal, char: CharProps(fontSize: 10)))380 s.append(Style(id: StyleID.tocHeading, name: "Contents Heading", basedOn: StyleID.normal, char: CharProps(bold: true)))381 s.append(Style(id: StyleID.toc(1), name: "Contents 1", basedOn: StyleID.normal))382 s.append(Style(id: StyleID.toc(2), name: "Contents 2", basedOn: StyleID.normal, para: ParaProps(indentLeft: 16)))383 s.append(Style(id: StyleID.bibliography, name: "Bibliography", basedOn: StyleID.normal, para: ParaProps(indentLeft: 24, firstLineIndent: -24)))384 s.append(Style(id: StyleID.emphasis, name: "Emphasis", kind: .character, char: CharProps(italic: true)))385 s.append(Style(id: StyleID.hyperlink, name: "Hyperlink", kind: .character, char: CharProps(underline: true, color: RGBA(hex: 0x0000FF))))386 return StyleSheet(defaultChar: CharProps(fontFamily: "Georgia", fontSize: 11.5, color: RGBA(hex: 0x16181D), language: "en"),387 defaultPara: ParaProps(alignment: .left, widowControl: true), styles: s, themeName: "Test")388}389390func testDocument(_ blocks: [Block]) -> DocumentModel {391 var d = DocumentModel(meta: DocumentMetadata(title: "Test", author: "Tester", subject: "Subject", keywords: ["one", "two"], language: "en",392 created: Date(timeIntervalSince1970: 1_780_000_000), modified: Date(timeIntervalSince1970: 1_780_003_600)),393 styles: representableStyles(), pageSetup: PageSetup(paper: .a4), body: blocks)394 d.settings.citationStyle = "apa"395 return d396}397398/// Identifier-insensitive equality of two models with a short report of the differences (never the whole model).399func expectSameModel(_ a: DocumentModel, _ b: DocumentModel, _ label: String = "", sourceLocation: SourceLocation = #_sourceLocation) {400 let na = normalized(a), nb = normalized(b)401 if na != nb {402 let details: String = modelDifferences(na, nb).joined(separator: "\n")403 Issue.record(Comment(rawValue: "models differ \(label):\n" + details), sourceLocation: sourceLocation)404 }405}406