Repository Public License
1// ORIX Convert — Packages/EngineDocuments/Tests/EngineDocumentsTests/TextAndInspectTests.swift2// Description: Text encodings and line endings, metadata carry-over and stripping, inspection of real document3// features, and file-specific preflight warnings on a DOCX assembled by editing its XML.4// Author: Simon-Pierre Boucher — contact@spboucher.ai5import Testing6import Foundation7import PDFKit8@testable import EngineDocuments9import ConvertCore1011@Suite("Text, metadata, inspection and preflight")12struct TextAndInspectTests {13 // MARK: Encodings1415 @Test func encodingDetection() throws {16 let sample = "Café déjà vu — naïve façade"17 func detect(_ data: Data, forced: String = "auto", html: Bool = false) throws -> DetectedText { try TextEncoding.decode(data, forced: forced, html: html) }18 var d = try detect(Data([0xEF, 0xBB, 0xBF]) + Data(sample.utf8))19 #expect(d.string == sample && d.encoding == .utf8 && d.method == "bom")20 d = try detect(Data([0xFF, 0xFE]) + sample.data(using: .utf16LittleEndian)!)21 #expect(d.string == sample && d.encoding == .utf16LittleEndian && d.method == "bom")22 d = try detect(Data([0xFE, 0xFF]) + sample.data(using: .utf16BigEndian)!)23 #expect(d.string == sample && d.encoding == .utf16BigEndian && d.method == "bom")24 d = try detect(Data([0xFF, 0xFE, 0, 0]) + sample.data(using: .utf32LittleEndian)!)25 #expect(d.string == sample && d.method == "bom")26 d = try detect(Data(sample.utf8))27 #expect(d.string == sample && d.encoding == .utf8 && d.method == "utf8" && !d.lossy)28 d = try detect(Data("plain ascii".utf8))29 #expect(d.encoding == .utf8 && d.method == "utf8")30 // Not UTF-8, no BOM: heuristics.31 let cp1252 = "Café déjà vu, naïve façade, élève, garçon, où êtes-vous ? Ça coûte 10 €.".data(using: .windowsCP1252)!32 #expect(String(data: cp1252, encoding: .utf8) == nil)33 d = try detect(cp1252)34 #expect(d.method == "heuristic" && d.string.contains("Café déjà vu") && d.string.contains("garçon"), "detected \(d.encodingName): \(d.string)")35 // HTML charset declaration wins over heuristics.36 let latin1 = "<html><head><meta charset=\"iso-8859-1\"></head><body>Élève réussi</body></html>".data(using: .isoLatin1)!37 d = try detect(latin1, html: true)38 #expect(d.method == "declared" && d.encoding == .isoLatin1 && d.string.contains("Élève réussi"))39 #expect(TextEncoding.htmlCharset(Data("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=Shift_JIS\">".utf8)) == "shift_jis")40 // Forcing.41 let sjis = "日本語のテキスト".data(using: .shiftJIS)!42 d = try detect(sjis, forced: "shift_jis")43 #expect(d.string == "日本語のテキスト" && d.method == "forced")44 #expect(throws: ConversionError.self) { _ = try detect(Data([0xC3, 0x28, 0xFF]), forced: "utf-8") }45 #expect(throws: ConversionError.self) { _ = try detect(Data(sample.utf8), forced: "klingon-8") }46 // Output side.47 #expect(TextEncoding.applyLineEnding("a\r\nb\rc\n", "crlf") == "a\r\nb\r\nc\r\n" && TextEncoding.applyLineEnding("a\r\nb", "cr") == "a\rb")48 let lossy = try TextEncoding.encode("price 10 € — 日本", as: "iso-8859-1")49 #expect(lossy.unrepresentable == 4, "€, —, 日, 本 do not exist in Latin-1: \(lossy.unrepresentable)")50 let bom = try TextEncoding.encode("x", as: "utf-8", bom: true)51 #expect(bom.data == Data([0xEF, 0xBB, 0xBF, 0x78]))52 }5354 @Test func encodingsThroughTheEngine() async throws {55 let dir = try Fixtures.newDirectory("encoding")56 let text = "Première ligne élève\nDeuxième ligne garçon naïf\n\nTroisième paragraphe où l'été coûte 10 €.\n"57 // A Windows-1252 text file with CRLF becomes a correct DOCX and a UTF-8 Markdown file.58 let legacy = dir.appendingPathComponent("legacy.txt")59 try text.replacingOccurrences(of: "\n", with: "\r\n").data(using: .windowsCP1252)!.write(to: legacy)60 let warnings = Engines.documents.preflight(ConversionRequest(source: legacy, sourceFormat: .txt, destinationFormat: .docx), inspection: nil)61 #expect(warnings.contains { $0.code == "encoding-guessed" })62 for engine in ["libreoffice", "apple", "pandoc"] {63 let docx = try await convert(legacy, .txt, .docx, options: ["engine": .string(engine)])64 #expect(docx.failedChecks.isEmpty, "\(engine): \(docx.failedChecks)")65 let f = try facts(docx.file, .docx)66 #expect(f.text.contains("Première ligne élève") && f.text.contains("garçon naïf") && f.text.contains("10 €"), "\(engine): \(f.text)")67 #expect(docx.recorder.has("encoding-detected"))68 }69 let md = try await convert(legacy, .txt, .markdown)70 let mdText = try String(contentsOf: md.file, encoding: .utf8)71 #expect(mdText.contains("Première ligne élève") && !mdText.contains("\r"))72 // Markdown is not interpreted in plain text input: characters stay literal.73 let tricky = dir.appendingPathComponent("tricky.txt")74 try Data("# not a heading\n* not a bullet *star*\n<b>not html</b> & [not](link)\n".utf8).write(to: tricky)75 let html = try await convert(tricky, .txt, .html)76 let scan = HTMLScanner.scan(try String(contentsOf: html.file, encoding: .utf8))77 #expect(scan.headings.isEmpty && !scan.hasBold && scan.links.isEmpty && scan.text.contains("# not a heading") && scan.text.contains("<b>not html</b> & [not](link)"))78 // UTF-16 with BOM in; forced encoding in.79 let utf16 = dir.appendingPathComponent("utf16.txt")80 try (Data([0xFF, 0xFE]) + text.data(using: .utf16LittleEndian)!).write(to: utf16)81 let fromUTF16 = try await convert(utf16, .txt, .rtf)82 let rtfFacts = try facts(fromUTF16.file, .rtf)83 #expect(rtfFacts.text.contains("Deuxième ligne garçon"))84 let sjis = dir.appendingPathComponent("sjis.txt")85 try "日本語のテキストです。\n".data(using: .shiftJIS)!.write(to: sjis)86 let forced = try await convert(sjis, .txt, .html, options: ["encoding.input": .string("shift_jis")])87 let forcedHTML = try String(contentsOf: forced.file, encoding: .utf8)88 #expect(forcedHTML.contains("日本語のテキストです。"))89 // Text output: encoding, BOM, line endings, and a warning for characters the encoding cannot hold.90 let fx = try await FixtureStore.shared.get()91 let out = try await convert(utf16, .txt, .markdown, options: ["text.lineending": .string("crlf")])92 let outData = try Data(contentsOf: out.file)93 let crlfText = String(decoding: outData, as: UTF8.self)94 #expect(crlfText.contains("élève") && crlfText.contains("\r\n") && !crlfText.replacingOccurrences(of: "\r\n", with: "").contains("\n"))95 let latin = try await convert(fx.documents[.docx]!, .docx, .txt, options: ["encoding.output": .string("utf-16le"), "encoding.bom": .bool(true), "text.lineending": .string("crlf")])96 #expect(latin.failedChecks.isEmpty, "\(latin.failedChecks)")97 let latinData = try Data(contentsOf: latin.file)98 #expect(latinData.prefix(2) == Data([0xFF, 0xFE]))99 let decoded = try #require(String(data: latinData.dropFirst(2), encoding: .utf16LittleEndian))100 #expect(decoded.contains("zebrafish") && decoded.contains("\r\n") && !decoded.replacingOccurrences(of: "\r\n", with: "").contains("\n"))101 let euro = dir.appendingPathComponent("euro.md")102 try Data("Prix : 10 € — 日本\n".utf8).write(to: euro)103 let narrowed = try await convert(euro, .markdown, .txt, options: ["encoding.output": .string("iso-8859-1")])104 #expect(narrowed.recorder.has("encoding-unrepresentable"))105 let narrowedText = try #require(String(data: try Data(contentsOf: narrowed.file), encoding: .isoLatin1))106 #expect(narrowedText.contains("Prix : 10"))107 // HTML with a declared legacy charset → DOCX keeps the accents.108 let page = dir.appendingPathComponent("latin1.html")109 try "<html><head><meta charset=\"iso-8859-1\"><title>Été</title></head><body><p>Élève réussi à l'école</p></body></html>".data(using: .isoLatin1)!.write(to: page)110 let pageDocx = try await convert(page, .html, .docx)111 let pageFacts = try facts(pageDocx.file, .docx)112 #expect(pageFacts.text.contains("Élève réussi à l'école") && pageFacts.title == "Été")113 let pagePDF = try await convert(page, .html, .pdf)114 let pagePDFFacts = try facts(pagePDF.file, .pdf)115 #expect(pagePDFFacts.text.precomposedStringWithCanonicalMapping.contains("Élève réussi") && pagePDFFacts.title == "Été")116 }117118 // MARK: Metadata119120 @Test func metadataIsCarriedOrStripped() async throws {121 let fx = try await FixtureStore.shared.get()122 let docx = fx.documents[.docx]!123 let src = try facts(docx, .docx)124 #expect(src.title == Probe.title && src.author == Probe.author && src.subject == "Fixture subject" && src.keywords == ["alpha", "beta"])125 for (d, engine) in [(FormatID.odt, "auto"), (.pdf, "auto"), (.rtf, "auto"), (.html, "auto"), (.html, "libreoffice"), (.docx, "x"), (.odt, "pandoc"), (.odt, "apple")] {126 let source = d == .docx ? fx.documents[.odt]! : docx127 let s: FormatID = d == .docx ? .odt : .docx128 let options: [String: OptionValue] = engine == "x" ? [:] : ["engine": .string(engine)]129 let kept = try await convert(source, s, d, options: options)130 let keptFacts = try facts(kept.file, d)131 #expect(keptFacts.title == Probe.title, "\(d) [\(engine)] carries the title: \(keptFacts.title ?? "nil")")132 var stripOptions = options; stripOptions["metadata.strip"] = .bool(true)133 let stripped = try await convert(source, s, d, options: stripOptions)134 #expect(stripped.failedChecks.isEmpty, "\(d) [\(engine)] stripped output validates: \(stripped.failedChecks)")135 let f = try facts(stripped.file, d)136 #expect((f.title ?? "").isEmpty || f.title == "master", "\(d) [\(engine)] title stripped: \(f.title ?? "nil")")137 #expect(f.author == nil && f.subject == nil && f.keywords.isEmpty, "\(d) [\(engine)] author / subject / keywords stripped: \(f.author ?? "nil")")138 if d != .html || engine != "auto" { #expect(normalized(f.searchText).contains("xylophone"), "\(d) [\(engine)] content is intact") }139 // The option is offered.140 let path = try #require(Engines.all.flatMap(\.paths).first { $0.source == s && $0.destination == d && $0.operation == nil })141 #expect(Engines.owning(path).options(for: path).contains { $0.id == "metadata.strip" })142 }143 // PDF from Markdown through WebKit: metadata set from the source, or left out.144 let pdf = try await convert(fx.documents[.markdown]!, .markdown, .pdf)145 let pdfFacts = try facts(pdf.file, .pdf)146 #expect(pdfFacts.title == Probe.title && pdfFacts.author == Probe.author && pdfFacts.keywords == ["alpha", "beta"])147 let bare = try await convert(fx.documents[.markdown]!, .markdown, .pdf, options: ["metadata.strip": .bool(true)])148 let bareFacts = try facts(bare.file, .pdf)149 #expect(bareFacts.title == nil && bareFacts.author == nil)150 }151152 // MARK: PDF options153154 @Test func pdfOptions() async throws {155 let fx = try await FixtureStore.shared.get()156 let dir = try Fixtures.newDirectory("pdfopt")157 // A long Markdown document: real pagination, paper size, margins, orientation.158 var long = "---\ntitle: Long Document\n---\n\n"159 for i in 1...120 { long += "## Section \(i)\n\nParagraph \(i) quetzal lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor.\n\n" }160 let md = dir.appendingPathComponent("long.md")161 try Data(long.utf8).write(to: md)162 let a4 = try await convert(md, .markdown, .pdf)163 #expect(a4.failedChecks.isEmpty, "\(a4.failedChecks)")164 let a4pdf = try #require(PDFDocument(url: a4.file))165 let a4box = try #require(a4pdf.page(at: 0)).bounds(for: .mediaBox)166 #expect(a4pdf.pageCount > 3 && abs(a4box.width - 595.28) < 1 && abs(a4box.height - 841.89) < 1, "A4 pages: \(a4pdf.pageCount) × \(a4box)")167 #expect((a4pdf.page(at: a4pdf.pageCount - 1)?.string ?? "").contains("Paragraph 120"), "the last page holds the end of the text")168 let letter = try await convert(md, .markdown, .pdf, options: ["page.size": .string("letter"), "page.orientation": .string("landscape"), "page.margin": .double(40)])169 let letterPDF = try #require(PDFDocument(url: letter.file))170 let box = try #require(letterPDF.page(at: 0)).bounds(for: .mediaBox)171 #expect(abs(box.width - 792) < 1 && abs(box.height - 612) < 1, "US Letter landscape: \(box)")172 #expect(letterPDF.pageCount != a4pdf.pageCount, "wider margins and another paper change the page count (\(letterPDF.pageCount) vs \(a4pdf.pageCount))")173 let continuous = try await convert(md, .markdown, .pdf, options: ["pdf.pagination": .string("continuous")])174 let one = try #require(PDFDocument(url: continuous.file))175 #expect(one.pageCount == 1 && (one.page(at: 0)?.bounds(for: .mediaBox).height ?? 0) > 3000 && (one.string ?? "").contains("Paragraph 120"))176 // LibreOffice: page range and PDF/A.177 let docxLong = dir.appendingPathComponent("long.docx")178 _ = try await Pandoc.run(["-f", "markdown", "-t", "docx", "-o", docxLong.path, md.path], scratch: dir)179 let full = try await convert(docxLong, .docx, .pdf)180 let fullPages = try #require(PDFDocument(url: full.file)).pageCount181 #expect(fullPages > 3)182 let range = try await convert(docxLong, .docx, .pdf, options: ["pdf.pages": .string("2-3")])183 #expect(PDFDocument(url: range.file)?.pageCount == 2)184 let archival = try await convert(fx.documents[.docx]!, .docx, .pdf, options: ["pdf.archival": .bool(true)])185 let bytes = try Data(contentsOf: archival.file)186 #expect(bytes.range(of: Data("pdfaid:part".utf8)) != nil, "PDF/A identification is present in the XMP metadata")187 await #expect(throws: ConversionError.self) { _ = try await convert(docxLong, .docx, .pdf, options: ["pdf.pages": .string("two")]) }188 // Markdown flavours.189 let gfm = try await convert(fx.documents[.docx]!, .docx, .markdown, options: ["markdown.flavor": .string("gfm")])190 let gfmText = try String(contentsOf: gfm.file, encoding: .utf8)191 #expect(gfmText.contains("| Kumquat") && gfmText.contains("# Chapter One Heading"), "GitHub pipe table")192 }193194 // MARK: Inspection and preflight on a DOCX assembled by editing its XML195196 static func featureDocx() async throws -> URL {197 let fx = try await FixtureStore.shared.get()198 let out = try Fixtures.newDirectory("features").appendingPathComponent("features.docx")199 let date = "w:author=\"Reviewer\" w:date=\"2026-01-02T03:04:05Z\""200 let extra = """201 <w:p><w:r><w:t xml:space="preserve">Kept text </w:t></w:r><w:ins w:id="901" \(date)><w:r><w:t>insertedword</w:t></w:r></w:ins><w:del w:id="902" \(date)><w:r><w:delText>deletedword</w:delText></w:r></w:del><w:r><w:commentReference w:id="0"/></w:r></w:p>202 <w:p><w:r><w:rPr><w:rFonts w:ascii="Zyxxy Imaginary Serif" w:hAnsi="Zyxxy Imaginary Serif"/></w:rPr><w:t>Text in a font nobody has.</w:t></w:r></w:p>203 <w:p><w:r><w:pict xmlns:v="urn:schemas-microsoft-com:vml"><v:shape style="width:120pt;height:40pt"><v:textbox><w:txbxContent><w:p><w:r><w:t>boxedtext</w:t></w:r></w:p></w:txbxContent></v:textbox></v:shape></w:pict></w:r></w:p>204 """205 try repackage(fx.documents[.docx]!, to: out, edits: [206 "word/vbaProject.bin": Data([0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1] + [UInt8](repeating: 0, count: 504)),207 "word/comments.xml": Data("<?xml version=\"1.0\" encoding=\"UTF-8\"?><w:comments xmlns:w=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\"><w:comment w:id=\"0\" w:author=\"Reviewer\" w:date=\"2026-01-02T03:04:05Z\"><w:p><w:r><w:t>commenttext</w:t></w:r></w:p></w:comment></w:comments>".utf8),208 "word/header1.xml": Data("<?xml version=\"1.0\" encoding=\"UTF-8\"?><w:hdr xmlns:w=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\"><w:p><w:r><w:t>Running header text</w:t></w:r></w:p></w:hdr>".utf8),209 ], transform: [210 "word/document.xml": { xml in211 guard let r = xml.range(of: "<w:body>") else { return xml }212 return xml.replacingCharacters(in: r, with: "<w:body>" + extra)213 },214 "word/_rels/document.xml.rels": { $0.replacingOccurrences(of: "</Relationships>", with: "<Relationship Id=\"rIdComments\" Type=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships/comments\" Target=\"comments.xml\"/><Relationship Id=\"rIdVba\" Type=\"http://schemas.microsoft.com/office/2006/relationships/vbaProject\" Target=\"vbaProject.bin\"/></Relationships>") },215 "[Content_Types].xml": { $0.replacingOccurrences(of: "</Types>", with: "<Default Extension=\"bin\" ContentType=\"application/vnd.ms-office.vbaProject\"/><Override PartName=\"/word/comments.xml\" ContentType=\"application/vnd.openxmlformats-officedocument.wordprocessingml.comments+xml\"/><Override PartName=\"/word/header1.xml\" ContentType=\"application/vnd.openxmlformats-officedocument.wordprocessingml.header+xml\"/></Types>") },216 ])217 return out218 }219220 @Test func inspectionReadsRealFeatures() async throws {221 let fx = try await FixtureStore.shared.get()222 let docx = try await Self.featureDocx()223 let i = try await Engines.documents.inspect(docx, format: .docx)224 func field(_ key: String, in inspection: FileInspection? = nil) -> String? { (inspection ?? i).fields.first { $0.key == key }?.value }225 #expect(field("Title") == Probe.title && field("Author") == Probe.author && field("Keywords") == "alpha, beta" && field("Language") == "en-CA")226 #expect(field("Macros")?.hasPrefix("Present") == true && field("Tracked changes") == "2" && field("Comments") == "1")227 #expect(field("Equations") == "2" && field("Text boxes") == "1" && field("Headers / footers with content") == "1")228 #expect(field("Tables") == "1" && field("Images") == "1" && field("Footnotes") == "1" && field("Links") == "1")229 #expect(field("Fonts")?.contains("Zyxxy Imaginary Serif") == true)230 #expect((Int(field("Words") ?? "") ?? 0) > 40 && field("Headings") == "3")231 #expect(i.hasMetadata == true && i.isEncrypted != true && i.pageCount == nil, "page count is not computed for a reflowable source")232 // The deleted text is not counted as content; the inserted text is.233 let f = try facts(docx, .docx)234 #expect(f.text.contains("insertedword") && !f.text.contains("deletedword") && f.text.contains("boxedtext"))235236 let odt = try await Engines.documents.inspect(fx.documents[.odt]!, format: .odt)237 #expect(field("Title", in: odt) == Probe.title, "odt title \(field("Title", in: odt) ?? "nil")")238 #expect(field("Tables", in: odt) == "1" && field("Headings", in: odt) == "3", "odt tables \(field("Tables", in: odt) ?? "nil") headings \(field("Headings", in: odt) ?? "nil")")239 #expect(field("Images", in: odt) == "1" && field("Equations", in: odt) == "2", "odt images \(field("Images", in: odt) ?? "nil") equations \(field("Equations", in: odt) ?? "nil") objects \(field("Embedded objects", in: odt) ?? "nil")")240 let html = try await Engines.documents.inspect(fx.documents[.html]!, format: .html)241 #expect(field("Title", in: html) == Probe.title && field("Author", in: html) == Probe.author && field("Language", in: html) == "en-CA" && field("Headings", in: html) == "4")242 let md = try await Engines.documents.inspect(fx.documents[.markdown]!, format: .markdown)243 #expect(field("Title", in: md) == Probe.title && field("Tables", in: md) == "1" && field("Images", in: md) == "1" && field("Headings", in: md) == "3" && field("Footnotes", in: md) == "1")244 let tex = try await Engines.documents.inspect(fx.documents[.latex]!, format: .latex)245 #expect(field("Title", in: tex) == Probe.title && field("Headings", in: tex) == "3" && field("Tables", in: tex) == "1")246 let rst = try await Engines.documents.inspect(fx.documents[.rst]!, format: .rst)247 #expect(field("Tables", in: rst) == "1" && field("Images", in: rst) == "1")248 let doc = try await Engines.documents.inspect(fx.documents[.doc]!, format: .doc)249 #expect(field("Title", in: doc) == Probe.title && field("Macros", in: doc) == "None" && field("Headings", in: doc) == nil, "headings cannot be told apart in a DOC: nothing is reported")250 let rtf = try await Engines.documents.inspect(fx.documents[.rtf]!, format: .rtf)251 #expect(field("Footnotes", in: rtf) == "1" && field("Tables", in: rtf) == "1")252 let txt = try await Engines.documents.inspect(fx.documents[.txt]!, format: .txt)253 #expect(field("Words", in: txt) == "19" && field("Title", in: txt) == nil, "txt words \(field("Words", in: txt) ?? "nil")")254 for engine in Engines.all { for format in engine.inspectableFormats { #expect(FormatCatalog.format(format) != nil) } }255 }256257 @Test func preflightWarnsAboutThisFile() async throws {258 let docx = try await Self.featureDocx()259 func codes(_ d: FormatID, _ options: [String: OptionValue] = [:]) -> [String: ConversionWarning] {260 let request = ConversionRequest(source: docx, sourceFormat: .docx, destinationFormat: d, options: ConversionOptions(options))261 let engine: any ConversionEngine = d == .epub ? Engines.ebooks : Engines.documents262 return Dictionary(engine.preflight(request, inspection: nil).map { ($0.code, $0) }, uniquingKeysWith: { a, _ in a })263 }264 let text = codes(.txt)265 for code in ["macros-dropped", "tracked-changes", "comments", "equations", "embedded-objects", "text-boxes", "footnotes", "headers-footers", "layout-reflowed"] where code != "embedded-objects" {266 #expect(text[code] != nil, "→ TXT warns about \(code): \(text.keys.sorted())")267 }268 #expect(text["macros-dropped"]?.severity == .dataLoss && text["macros-dropped"]?.message.contains("never executed") == true)269 #expect(text["headers-footers"]?.severity == .dataLoss && text["comments"]?.severity == .dataLoss)270 let markdown = codes(.markdown)271 #expect(markdown["headers-footers"] != nil && markdown["text-boxes"] != nil && markdown["footnotes"] == nil, "Markdown holds footnotes: no footnote warning")272 let pdf = codes(.pdf)273 #expect(pdf["fonts-missing"]?.message.contains("Zyxxy Imaginary Serif") == true, "the missing font is named: \(pdf["fonts-missing"]?.message ?? "none")")274 #expect(pdf["macros-dropped"] != nil && pdf["tracked-changes"] != nil && pdf["comments"]?.severity == .caution)275 #expect(pdf["headers-footers"] == nil && pdf["layout-reflowed"] == nil, "a PDF keeps the page layout")276 let odt = codes(.odt)277 #expect(odt["comments"] == nil && odt["fonts-missing"] != nil && odt["macros-dropped"] != nil)278 #expect(codes(.odt, ["engine": .string("apple")])["images-dropped"] != nil, "the Apple text system is announced to drop images")279 #expect(codes(.epub)["layout-reflowed"] != nil)280 // Fonts that exist here (system or bundled with LibreOffice) are not reported.281 #expect(Fonts.missing(from: ["Helvetica", "Liberation Serif", "Carlito", "Zyxxy Imaginary Serif", "serif"]) == ["Zyxxy Imaginary Serif"])282 #expect(Fonts.describeMissing(["Calibri"]).contains("Carlito"))283284 // And the conversions do what the warnings say.285 let md = try await convert(docx, .docx, .markdown)286 let mdText = try String(contentsOf: md.file, encoding: .utf8)287 #expect(mdText.contains("insertedword") && !mdText.contains("deletedword"), "insertions accepted, deletions removed")288 #expect(!mdText.contains("commenttext") && !mdText.contains("Running header text"), "comments and headers are not in the Markdown")289 #expect(md.recorder.has("macros-dropped") && md.recorder.has("headers-footers") && md.recorder.has("comments"))290 let odtRun = try await convert(docx, .docx, .odt)291 #expect(odtRun.failedChecks.isEmpty, "\(odtRun.failedChecks)")292 let zip = try ZipReader(url: odtRun.file)293 #expect(!zip.entries.contains { $0.name.hasPrefix("Basic/") || $0.name.lowercased().contains("vba") }, "no macro storage in the converted file")294 let odtFacts = try facts(odtRun.file, .odt)295 #expect(odtFacts.comments == 1 && odtFacts.trackedChanges >= 1, "LibreOffice keeps comments and change tracking in ODT")296 let pdfRun = try await convert(docx, .docx, .pdf)297 #expect(pdfRun.failedChecks.isEmpty && pdfRun.recorder.has("fonts-missing"))298 // LaTeX and Pages / Keynote.299 let fx = try await FixtureStore.shared.get()300 let tex = Engines.documents.preflight(ConversionRequest(source: fx.documents[.latex]!, sourceFormat: .latex, destinationFormat: .docx), inspection: nil)301 #expect(tex.contains { $0.code == "latex-not-typeset" })302 for format in [FormatID.pages, .keynote] {303 #expect(!Engines.all.contains { $0.inspectableFormats.contains(format) || $0.paths.contains { $0.source == format } }, "\(format) is documented as unsupported")304 }305 }306307 @Test func htmlSanitizer() {308 let html = """309 <html><head><base href="http://evil.example/"><link rel="stylesheet" href="https://cdn.example/a.css"><link rel="stylesheet" href="local.css">310 <style>@import url("http://evil.example/i.css"); p { background: url('//evil.example/b.png'); color: red; } div { background: url(img/ok.png) }</style></head>311 <body><a href="https://example.org/keep">link</a><img src="pic.png"><img src="HTTP://EVIL.example/x.png"><img src="/etc/passwd"><img src="../up.png">312 <img srcset="a.png 1x, https://evil.example/b.png 2x"><iframe src="inner.html"></iframe><video poster='ftp://evil.example/p.png'></video>313 <img src=data:image/png;base64,AAAA><svg><image xlink:href="file:///etc/hosts"/></svg></body></html>314 """315 let r = HTMLSanitizer.neutralize(html, confine: true)316 #expect(r.remote.count == 7, "remote (including the <base> target): \(r.remote)")317 #expect(Set(r.outside) == Set(["/etc/passwd", "../up.png", "inner.html", "file:///etc/hosts"]), "outside: \(r.outside)")318 #expect(!r.html.contains("evil.example") && !r.html.contains("/etc/") && !r.html.contains("inner.html") && !r.html.contains("../up.png"))319 #expect(r.html.contains("href=\"https://example.org/keep\"") && r.html.contains("src=\"pic.png\"") && r.html.contains("href=\"local.css\"") && r.html.contains("url(img/ok.png)") && r.html.contains("data:image/png;base64,AAAA") && r.html.contains("color: red"))320 let relaxed = HTMLSanitizer.neutralize("<img src=\"../images/a.png\"><img src=\"//x.example/a.png\">", confine: false)321 #expect(relaxed.html.contains("../images/a.png") && relaxed.remote == ["//x.example/a.png"])322 #expect(HTMLSanitizer.classify("C:\\\\Windows\\\\x.png", confine: true) == .outside && HTMLSanitizer.classify("a/b/../c.png", confine: true) == .keep && HTMLSanitizer.classify("#frag", confine: true) == .keep)323 }324}325