SPB Git

Repository Public License

TextAndInspectTests.swift 27.1 KB 325 lines swift

1// ORIX Convert — Packages/EngineDocuments/Tests/EngineDocumentsTests/TextAndInspectTests.swift2// Description: Text encodings and line endings, metadata carry-over and stripping, inspection of real document3//              features, and file-specific preflight warnings on a DOCX assembled by editing its XML.4// Author: Simon-Pierre Boucher — contact@spboucher.ai5import Testing6import Foundation7import PDFKit8@testable import EngineDocuments9import ConvertCore1011@Suite("Text, metadata, inspection and preflight")12struct TextAndInspectTests {13    // MARK: Encodings1415    @Test func encodingDetection() throws {16        let sample = "Café déjà vu — naïve façade"17        func detect(_ data: Data, forced: String = "auto", html: Bool = false) throws -> DetectedText { try TextEncoding.decode(data, forced: forced, html: html) }18        var d = try detect(Data([0xEF, 0xBB, 0xBF]) + Data(sample.utf8))19        #expect(d.string == sample && d.encoding == .utf8 && d.method == "bom")20        d = try detect(Data([0xFF, 0xFE]) + sample.data(using: .utf16LittleEndian)!)21        #expect(d.string == sample && d.encoding == .utf16LittleEndian && d.method == "bom")22        d = try detect(Data([0xFE, 0xFF]) + sample.data(using: .utf16BigEndian)!)23        #expect(d.string == sample && d.encoding == .utf16BigEndian && d.method == "bom")24        d = try detect(Data([0xFF, 0xFE, 0, 0]) + sample.data(using: .utf32LittleEndian)!)25        #expect(d.string == sample && d.method == "bom")26        d = try detect(Data(sample.utf8))27        #expect(d.string == sample && d.encoding == .utf8 && d.method == "utf8" && !d.lossy)28        d = try detect(Data("plain ascii".utf8))29        #expect(d.encoding == .utf8 && d.method == "utf8")30        // Not UTF-8, no BOM: heuristics.31        let cp1252 = "Café déjà vu, naïve façade, élève, garçon, où êtes-vous ? Ça coûte 10 €.".data(using: .windowsCP1252)!32        #expect(String(data: cp1252, encoding: .utf8) == nil)33        d = try detect(cp1252)34        #expect(d.method == "heuristic" && d.string.contains("Café déjà vu") && d.string.contains("garçon"), "detected \(d.encodingName): \(d.string)")35        // HTML charset declaration wins over heuristics.36        let latin1 = "<html><head><meta charset=\"iso-8859-1\"></head><body>Élève réussi</body></html>".data(using: .isoLatin1)!37        d = try detect(latin1, html: true)38        #expect(d.method == "declared" && d.encoding == .isoLatin1 && d.string.contains("Élève réussi"))39        #expect(TextEncoding.htmlCharset(Data("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=Shift_JIS\">".utf8)) == "shift_jis")40        // Forcing.41        let sjis = "日本語のテキスト".data(using: .shiftJIS)!42        d = try detect(sjis, forced: "shift_jis")43        #expect(d.string == "日本語のテキスト" && d.method == "forced")44        #expect(throws: ConversionError.self) { _ = try detect(Data([0xC3, 0x28, 0xFF]), forced: "utf-8") }45        #expect(throws: ConversionError.self) { _ = try detect(Data(sample.utf8), forced: "klingon-8") }46        // Output side.47        #expect(TextEncoding.applyLineEnding("a\r\nb\rc\n", "crlf") == "a\r\nb\r\nc\r\n" && TextEncoding.applyLineEnding("a\r\nb", "cr") == "a\rb")48        let lossy = try TextEncoding.encode("price 10 € — 日本", as: "iso-8859-1")49        #expect(lossy.unrepresentable == 4, "€, —, 日, 本 do not exist in Latin-1: \(lossy.unrepresentable)")50        let bom = try TextEncoding.encode("x", as: "utf-8", bom: true)51        #expect(bom.data == Data([0xEF, 0xBB, 0xBF, 0x78]))52    }5354    @Test func encodingsThroughTheEngine() async throws {55        let dir = try Fixtures.newDirectory("encoding")56        let text = "Première ligne élève\nDeuxième ligne garçon naïf\n\nTroisième paragraphe où l'été coûte 10 €.\n"57        // A Windows-1252 text file with CRLF becomes a correct DOCX and a UTF-8 Markdown file.58        let legacy = dir.appendingPathComponent("legacy.txt")59        try text.replacingOccurrences(of: "\n", with: "\r\n").data(using: .windowsCP1252)!.write(to: legacy)60        let warnings = Engines.documents.preflight(ConversionRequest(source: legacy, sourceFormat: .txt, destinationFormat: .docx), inspection: nil)61        #expect(warnings.contains { $0.code == "encoding-guessed" })62        for engine in ["libreoffice", "apple", "pandoc"] {63            let docx = try await convert(legacy, .txt, .docx, options: ["engine": .string(engine)])64            #expect(docx.failedChecks.isEmpty, "\(engine): \(docx.failedChecks)")65            let f = try facts(docx.file, .docx)66            #expect(f.text.contains("Première ligne élève") && f.text.contains("garçon naïf") && f.text.contains("10 €"), "\(engine): \(f.text)")67            #expect(docx.recorder.has("encoding-detected"))68        }69        let md = try await convert(legacy, .txt, .markdown)70        let mdText = try String(contentsOf: md.file, encoding: .utf8)71        #expect(mdText.contains("Première ligne élève") && !mdText.contains("\r"))72        // Markdown is not interpreted in plain text input: characters stay literal.73        let tricky = dir.appendingPathComponent("tricky.txt")74        try Data("# not a heading\n* not a bullet *star*\n<b>not html</b> & [not](link)\n".utf8).write(to: tricky)75        let html = try await convert(tricky, .txt, .html)76        let scan = HTMLScanner.scan(try String(contentsOf: html.file, encoding: .utf8))77        #expect(scan.headings.isEmpty && !scan.hasBold && scan.links.isEmpty && scan.text.contains("# not a heading") && scan.text.contains("<b>not html</b> & [not](link)"))78        // UTF-16 with BOM in; forced encoding in.79        let utf16 = dir.appendingPathComponent("utf16.txt")80        try (Data([0xFF, 0xFE]) + text.data(using: .utf16LittleEndian)!).write(to: utf16)81        let fromUTF16 = try await convert(utf16, .txt, .rtf)82        let rtfFacts = try facts(fromUTF16.file, .rtf)83        #expect(rtfFacts.text.contains("Deuxième ligne garçon"))84        let sjis = dir.appendingPathComponent("sjis.txt")85        try "日本語のテキストです。\n".data(using: .shiftJIS)!.write(to: sjis)86        let forced = try await convert(sjis, .txt, .html, options: ["encoding.input": .string("shift_jis")])87        let forcedHTML = try String(contentsOf: forced.file, encoding: .utf8)88        #expect(forcedHTML.contains("日本語のテキストです。"))89        // Text output: encoding, BOM, line endings, and a warning for characters the encoding cannot hold.90        let fx = try await FixtureStore.shared.get()91        let out = try await convert(utf16, .txt, .markdown, options: ["text.lineending": .string("crlf")])92        let outData = try Data(contentsOf: out.file)93        let crlfText = String(decoding: outData, as: UTF8.self)94        #expect(crlfText.contains("élève") && crlfText.contains("\r\n") && !crlfText.replacingOccurrences(of: "\r\n", with: "").contains("\n"))95        let latin = try await convert(fx.documents[.docx]!, .docx, .txt, options: ["encoding.output": .string("utf-16le"), "encoding.bom": .bool(true), "text.lineending": .string("crlf")])96        #expect(latin.failedChecks.isEmpty, "\(latin.failedChecks)")97        let latinData = try Data(contentsOf: latin.file)98        #expect(latinData.prefix(2) == Data([0xFF, 0xFE]))99        let decoded = try #require(String(data: latinData.dropFirst(2), encoding: .utf16LittleEndian))100        #expect(decoded.contains("zebrafish") && decoded.contains("\r\n") && !decoded.replacingOccurrences(of: "\r\n", with: "").contains("\n"))101        let euro = dir.appendingPathComponent("euro.md")102        try Data("Prix : 10 € — 日本\n".utf8).write(to: euro)103        let narrowed = try await convert(euro, .markdown, .txt, options: ["encoding.output": .string("iso-8859-1")])104        #expect(narrowed.recorder.has("encoding-unrepresentable"))105        let narrowedText = try #require(String(data: try Data(contentsOf: narrowed.file), encoding: .isoLatin1))106        #expect(narrowedText.contains("Prix : 10"))107        // HTML with a declared legacy charset → DOCX keeps the accents.108        let page = dir.appendingPathComponent("latin1.html")109        try "<html><head><meta charset=\"iso-8859-1\"><title>Été</title></head><body><p>Élève réussi à l'école</p></body></html>".data(using: .isoLatin1)!.write(to: page)110        let pageDocx = try await convert(page, .html, .docx)111        let pageFacts = try facts(pageDocx.file, .docx)112        #expect(pageFacts.text.contains("Élève réussi à l'école") && pageFacts.title == "Été")113        let pagePDF = try await convert(page, .html, .pdf)114        let pagePDFFacts = try facts(pagePDF.file, .pdf)115        #expect(pagePDFFacts.text.precomposedStringWithCanonicalMapping.contains("Élève réussi") && pagePDFFacts.title == "Été")116    }117118    // MARK: Metadata119120    @Test func metadataIsCarriedOrStripped() async throws {121        let fx = try await FixtureStore.shared.get()122        let docx = fx.documents[.docx]!123        let src = try facts(docx, .docx)124        #expect(src.title == Probe.title && src.author == Probe.author && src.subject == "Fixture subject" && src.keywords == ["alpha", "beta"])125        for (d, engine) in [(FormatID.odt, "auto"), (.pdf, "auto"), (.rtf, "auto"), (.html, "auto"), (.html, "libreoffice"), (.docx, "x"), (.odt, "pandoc"), (.odt, "apple")] {126            let source = d == .docx ? fx.documents[.odt]! : docx127            let s: FormatID = d == .docx ? .odt : .docx128            let options: [String: OptionValue] = engine == "x" ? [:] : ["engine": .string(engine)]129            let kept = try await convert(source, s, d, options: options)130            let keptFacts = try facts(kept.file, d)131            #expect(keptFacts.title == Probe.title, "\(d) [\(engine)] carries the title: \(keptFacts.title ?? "nil")")132            var stripOptions = options; stripOptions["metadata.strip"] = .bool(true)133            let stripped = try await convert(source, s, d, options: stripOptions)134            #expect(stripped.failedChecks.isEmpty, "\(d) [\(engine)] stripped output validates: \(stripped.failedChecks)")135            let f = try facts(stripped.file, d)136            #expect((f.title ?? "").isEmpty || f.title == "master", "\(d) [\(engine)] title stripped: \(f.title ?? "nil")")137            #expect(f.author == nil && f.subject == nil && f.keywords.isEmpty, "\(d) [\(engine)] author / subject / keywords stripped: \(f.author ?? "nil")")138            if d != .html || engine != "auto" { #expect(normalized(f.searchText).contains("xylophone"), "\(d) [\(engine)] content is intact") }139            // The option is offered.140            let path = try #require(Engines.all.flatMap(\.paths).first { $0.source == s && $0.destination == d && $0.operation == nil })141            #expect(Engines.owning(path).options(for: path).contains { $0.id == "metadata.strip" })142        }143        // PDF from Markdown through WebKit: metadata set from the source, or left out.144        let pdf = try await convert(fx.documents[.markdown]!, .markdown, .pdf)145        let pdfFacts = try facts(pdf.file, .pdf)146        #expect(pdfFacts.title == Probe.title && pdfFacts.author == Probe.author && pdfFacts.keywords == ["alpha", "beta"])147        let bare = try await convert(fx.documents[.markdown]!, .markdown, .pdf, options: ["metadata.strip": .bool(true)])148        let bareFacts = try facts(bare.file, .pdf)149        #expect(bareFacts.title == nil && bareFacts.author == nil)150    }151152    // MARK: PDF options153154    @Test func pdfOptions() async throws {155        let fx = try await FixtureStore.shared.get()156        let dir = try Fixtures.newDirectory("pdfopt")157        // A long Markdown document: real pagination, paper size, margins, orientation.158        var long = "---\ntitle: Long Document\n---\n\n"159        for i in 1...120 { long += "## Section \(i)\n\nParagraph \(i) quetzal lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor.\n\n" }160        let md = dir.appendingPathComponent("long.md")161        try Data(long.utf8).write(to: md)162        let a4 = try await convert(md, .markdown, .pdf)163        #expect(a4.failedChecks.isEmpty, "\(a4.failedChecks)")164        let a4pdf = try #require(PDFDocument(url: a4.file))165        let a4box = try #require(a4pdf.page(at: 0)).bounds(for: .mediaBox)166        #expect(a4pdf.pageCount > 3 && abs(a4box.width - 595.28) < 1 && abs(a4box.height - 841.89) < 1, "A4 pages: \(a4pdf.pageCount) × \(a4box)")167        #expect((a4pdf.page(at: a4pdf.pageCount - 1)?.string ?? "").contains("Paragraph 120"), "the last page holds the end of the text")168        let letter = try await convert(md, .markdown, .pdf, options: ["page.size": .string("letter"), "page.orientation": .string("landscape"), "page.margin": .double(40)])169        let letterPDF = try #require(PDFDocument(url: letter.file))170        let box = try #require(letterPDF.page(at: 0)).bounds(for: .mediaBox)171        #expect(abs(box.width - 792) < 1 && abs(box.height - 612) < 1, "US Letter landscape: \(box)")172        #expect(letterPDF.pageCount != a4pdf.pageCount, "wider margins and another paper change the page count (\(letterPDF.pageCount) vs \(a4pdf.pageCount))")173        let continuous = try await convert(md, .markdown, .pdf, options: ["pdf.pagination": .string("continuous")])174        let one = try #require(PDFDocument(url: continuous.file))175        #expect(one.pageCount == 1 && (one.page(at: 0)?.bounds(for: .mediaBox).height ?? 0) > 3000 && (one.string ?? "").contains("Paragraph 120"))176        // LibreOffice: page range and PDF/A.177        let docxLong = dir.appendingPathComponent("long.docx")178        _ = try await Pandoc.run(["-f", "markdown", "-t", "docx", "-o", docxLong.path, md.path], scratch: dir)179        let full = try await convert(docxLong, .docx, .pdf)180        let fullPages = try #require(PDFDocument(url: full.file)).pageCount181        #expect(fullPages > 3)182        let range = try await convert(docxLong, .docx, .pdf, options: ["pdf.pages": .string("2-3")])183        #expect(PDFDocument(url: range.file)?.pageCount == 2)184        let archival = try await convert(fx.documents[.docx]!, .docx, .pdf, options: ["pdf.archival": .bool(true)])185        let bytes = try Data(contentsOf: archival.file)186        #expect(bytes.range(of: Data("pdfaid:part".utf8)) != nil, "PDF/A identification is present in the XMP metadata")187        await #expect(throws: ConversionError.self) { _ = try await convert(docxLong, .docx, .pdf, options: ["pdf.pages": .string("two")]) }188        // Markdown flavours.189        let gfm = try await convert(fx.documents[.docx]!, .docx, .markdown, options: ["markdown.flavor": .string("gfm")])190        let gfmText = try String(contentsOf: gfm.file, encoding: .utf8)191        #expect(gfmText.contains("| Kumquat") && gfmText.contains("# Chapter One Heading"), "GitHub pipe table")192    }193194    // MARK: Inspection and preflight on a DOCX assembled by editing its XML195196    static func featureDocx() async throws -> URL {197        let fx = try await FixtureStore.shared.get()198        let out = try Fixtures.newDirectory("features").appendingPathComponent("features.docx")199        let date = "w:author=\"Reviewer\" w:date=\"2026-01-02T03:04:05Z\""200        let extra = """201        <w:p><w:r><w:t xml:space="preserve">Kept text </w:t></w:r><w:ins w:id="901" \(date)><w:r><w:t>insertedword</w:t></w:r></w:ins><w:del w:id="902" \(date)><w:r><w:delText>deletedword</w:delText></w:r></w:del><w:r><w:commentReference w:id="0"/></w:r></w:p>202        <w:p><w:r><w:rPr><w:rFonts w:ascii="Zyxxy Imaginary Serif" w:hAnsi="Zyxxy Imaginary Serif"/></w:rPr><w:t>Text in a font nobody has.</w:t></w:r></w:p>203        <w:p><w:r><w:pict xmlns:v="urn:schemas-microsoft-com:vml"><v:shape style="width:120pt;height:40pt"><v:textbox><w:txbxContent><w:p><w:r><w:t>boxedtext</w:t></w:r></w:p></w:txbxContent></v:textbox></v:shape></w:pict></w:r></w:p>204        """205        try repackage(fx.documents[.docx]!, to: out, edits: [206            "word/vbaProject.bin": Data([0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1] + [UInt8](repeating: 0, count: 504)),207            "word/comments.xml": Data("<?xml version=\"1.0\" encoding=\"UTF-8\"?><w:comments xmlns:w=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\"><w:comment w:id=\"0\" w:author=\"Reviewer\" w:date=\"2026-01-02T03:04:05Z\"><w:p><w:r><w:t>commenttext</w:t></w:r></w:p></w:comment></w:comments>".utf8),208            "word/header1.xml": Data("<?xml version=\"1.0\" encoding=\"UTF-8\"?><w:hdr xmlns:w=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\"><w:p><w:r><w:t>Running header text</w:t></w:r></w:p></w:hdr>".utf8),209        ], transform: [210            "word/document.xml": { xml in211                guard let r = xml.range(of: "<w:body>") else { return xml }212                return xml.replacingCharacters(in: r, with: "<w:body>" + extra)213            },214            "word/_rels/document.xml.rels": { $0.replacingOccurrences(of: "</Relationships>", with: "<Relationship Id=\"rIdComments\" Type=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships/comments\" Target=\"comments.xml\"/><Relationship Id=\"rIdVba\" Type=\"http://schemas.microsoft.com/office/2006/relationships/vbaProject\" Target=\"vbaProject.bin\"/></Relationships>") },215            "[Content_Types].xml": { $0.replacingOccurrences(of: "</Types>", with: "<Default Extension=\"bin\" ContentType=\"application/vnd.ms-office.vbaProject\"/><Override PartName=\"/word/comments.xml\" ContentType=\"application/vnd.openxmlformats-officedocument.wordprocessingml.comments+xml\"/><Override PartName=\"/word/header1.xml\" ContentType=\"application/vnd.openxmlformats-officedocument.wordprocessingml.header+xml\"/></Types>") },216        ])217        return out218    }219220    @Test func inspectionReadsRealFeatures() async throws {221        let fx = try await FixtureStore.shared.get()222        let docx = try await Self.featureDocx()223        let i = try await Engines.documents.inspect(docx, format: .docx)224        func field(_ key: String, in inspection: FileInspection? = nil) -> String? { (inspection ?? i).fields.first { $0.key == key }?.value }225        #expect(field("Title") == Probe.title && field("Author") == Probe.author && field("Keywords") == "alpha, beta" && field("Language") == "en-CA")226        #expect(field("Macros")?.hasPrefix("Present") == true && field("Tracked changes") == "2" && field("Comments") == "1")227        #expect(field("Equations") == "2" && field("Text boxes") == "1" && field("Headers / footers with content") == "1")228        #expect(field("Tables") == "1" && field("Images") == "1" && field("Footnotes") == "1" && field("Links") == "1")229        #expect(field("Fonts")?.contains("Zyxxy Imaginary Serif") == true)230        #expect((Int(field("Words") ?? "") ?? 0) > 40 && field("Headings") == "3")231        #expect(i.hasMetadata == true && i.isEncrypted != true && i.pageCount == nil, "page count is not computed for a reflowable source")232        // The deleted text is not counted as content; the inserted text is.233        let f = try facts(docx, .docx)234        #expect(f.text.contains("insertedword") && !f.text.contains("deletedword") && f.text.contains("boxedtext"))235236        let odt = try await Engines.documents.inspect(fx.documents[.odt]!, format: .odt)237        #expect(field("Title", in: odt) == Probe.title, "odt title \(field("Title", in: odt) ?? "nil")")238        #expect(field("Tables", in: odt) == "1" && field("Headings", in: odt) == "3", "odt tables \(field("Tables", in: odt) ?? "nil") headings \(field("Headings", in: odt) ?? "nil")")239        #expect(field("Images", in: odt) == "1" && field("Equations", in: odt) == "2", "odt images \(field("Images", in: odt) ?? "nil") equations \(field("Equations", in: odt) ?? "nil") objects \(field("Embedded objects", in: odt) ?? "nil")")240        let html = try await Engines.documents.inspect(fx.documents[.html]!, format: .html)241        #expect(field("Title", in: html) == Probe.title && field("Author", in: html) == Probe.author && field("Language", in: html) == "en-CA" && field("Headings", in: html) == "4")242        let md = try await Engines.documents.inspect(fx.documents[.markdown]!, format: .markdown)243        #expect(field("Title", in: md) == Probe.title && field("Tables", in: md) == "1" && field("Images", in: md) == "1" && field("Headings", in: md) == "3" && field("Footnotes", in: md) == "1")244        let tex = try await Engines.documents.inspect(fx.documents[.latex]!, format: .latex)245        #expect(field("Title", in: tex) == Probe.title && field("Headings", in: tex) == "3" && field("Tables", in: tex) == "1")246        let rst = try await Engines.documents.inspect(fx.documents[.rst]!, format: .rst)247        #expect(field("Tables", in: rst) == "1" && field("Images", in: rst) == "1")248        let doc = try await Engines.documents.inspect(fx.documents[.doc]!, format: .doc)249        #expect(field("Title", in: doc) == Probe.title && field("Macros", in: doc) == "None" && field("Headings", in: doc) == nil, "headings cannot be told apart in a DOC: nothing is reported")250        let rtf = try await Engines.documents.inspect(fx.documents[.rtf]!, format: .rtf)251        #expect(field("Footnotes", in: rtf) == "1" && field("Tables", in: rtf) == "1")252        let txt = try await Engines.documents.inspect(fx.documents[.txt]!, format: .txt)253        #expect(field("Words", in: txt) == "19" && field("Title", in: txt) == nil, "txt words \(field("Words", in: txt) ?? "nil")")254        for engine in Engines.all { for format in engine.inspectableFormats { #expect(FormatCatalog.format(format) != nil) } }255    }256257    @Test func preflightWarnsAboutThisFile() async throws {258        let docx = try await Self.featureDocx()259        func codes(_ d: FormatID, _ options: [String: OptionValue] = [:]) -> [String: ConversionWarning] {260            let request = ConversionRequest(source: docx, sourceFormat: .docx, destinationFormat: d, options: ConversionOptions(options))261            let engine: any ConversionEngine = d == .epub ? Engines.ebooks : Engines.documents262            return Dictionary(engine.preflight(request, inspection: nil).map { ($0.code, $0) }, uniquingKeysWith: { a, _ in a })263        }264        let text = codes(.txt)265        for code in ["macros-dropped", "tracked-changes", "comments", "equations", "embedded-objects", "text-boxes", "footnotes", "headers-footers", "layout-reflowed"] where code != "embedded-objects" {266            #expect(text[code] != nil, "→ TXT warns about \(code): \(text.keys.sorted())")267        }268        #expect(text["macros-dropped"]?.severity == .dataLoss && text["macros-dropped"]?.message.contains("never executed") == true)269        #expect(text["headers-footers"]?.severity == .dataLoss && text["comments"]?.severity == .dataLoss)270        let markdown = codes(.markdown)271        #expect(markdown["headers-footers"] != nil && markdown["text-boxes"] != nil && markdown["footnotes"] == nil, "Markdown holds footnotes: no footnote warning")272        let pdf = codes(.pdf)273        #expect(pdf["fonts-missing"]?.message.contains("Zyxxy Imaginary Serif") == true, "the missing font is named: \(pdf["fonts-missing"]?.message ?? "none")")274        #expect(pdf["macros-dropped"] != nil && pdf["tracked-changes"] != nil && pdf["comments"]?.severity == .caution)275        #expect(pdf["headers-footers"] == nil && pdf["layout-reflowed"] == nil, "a PDF keeps the page layout")276        let odt = codes(.odt)277        #expect(odt["comments"] == nil && odt["fonts-missing"] != nil && odt["macros-dropped"] != nil)278        #expect(codes(.odt, ["engine": .string("apple")])["images-dropped"] != nil, "the Apple text system is announced to drop images")279        #expect(codes(.epub)["layout-reflowed"] != nil)280        // Fonts that exist here (system or bundled with LibreOffice) are not reported.281        #expect(Fonts.missing(from: ["Helvetica", "Liberation Serif", "Carlito", "Zyxxy Imaginary Serif", "serif"]) == ["Zyxxy Imaginary Serif"])282        #expect(Fonts.describeMissing(["Calibri"]).contains("Carlito"))283284        // And the conversions do what the warnings say.285        let md = try await convert(docx, .docx, .markdown)286        let mdText = try String(contentsOf: md.file, encoding: .utf8)287        #expect(mdText.contains("insertedword") && !mdText.contains("deletedword"), "insertions accepted, deletions removed")288        #expect(!mdText.contains("commenttext") && !mdText.contains("Running header text"), "comments and headers are not in the Markdown")289        #expect(md.recorder.has("macros-dropped") && md.recorder.has("headers-footers") && md.recorder.has("comments"))290        let odtRun = try await convert(docx, .docx, .odt)291        #expect(odtRun.failedChecks.isEmpty, "\(odtRun.failedChecks)")292        let zip = try ZipReader(url: odtRun.file)293        #expect(!zip.entries.contains { $0.name.hasPrefix("Basic/") || $0.name.lowercased().contains("vba") }, "no macro storage in the converted file")294        let odtFacts = try facts(odtRun.file, .odt)295        #expect(odtFacts.comments == 1 && odtFacts.trackedChanges >= 1, "LibreOffice keeps comments and change tracking in ODT")296        let pdfRun = try await convert(docx, .docx, .pdf)297        #expect(pdfRun.failedChecks.isEmpty && pdfRun.recorder.has("fonts-missing"))298        // LaTeX and Pages / Keynote.299        let fx = try await FixtureStore.shared.get()300        let tex = Engines.documents.preflight(ConversionRequest(source: fx.documents[.latex]!, sourceFormat: .latex, destinationFormat: .docx), inspection: nil)301        #expect(tex.contains { $0.code == "latex-not-typeset" })302        for format in [FormatID.pages, .keynote] {303            #expect(!Engines.all.contains { $0.inspectableFormats.contains(format) || $0.paths.contains { $0.source == format } }, "\(format) is documented as unsupported")304        }305    }306307    @Test func htmlSanitizer() {308        let html = """309        <html><head><base href="http://evil.example/"><link rel="stylesheet" href="https://cdn.example/a.css"><link rel="stylesheet" href="local.css">310        <style>@import url("http://evil.example/i.css"); p { background: url('//evil.example/b.png'); color: red; } div { background: url(img/ok.png) }</style></head>311        <body><a href="https://example.org/keep">link</a><img src="pic.png"><img src="HTTP://EVIL.example/x.png"><img src="/etc/passwd"><img src="../up.png">312        <img srcset="a.png 1x, https://evil.example/b.png 2x"><iframe src="inner.html"></iframe><video poster='ftp://evil.example/p.png'></video>313        <img src=data:image/png;base64,AAAA><svg><image xlink:href="file:///etc/hosts"/></svg></body></html>314        """315        let r = HTMLSanitizer.neutralize(html, confine: true)316        #expect(r.remote.count == 7, "remote (including the <base> target): \(r.remote)")317        #expect(Set(r.outside) == Set(["/etc/passwd", "../up.png", "inner.html", "file:///etc/hosts"]), "outside: \(r.outside)")318        #expect(!r.html.contains("evil.example") && !r.html.contains("/etc/") && !r.html.contains("inner.html") && !r.html.contains("../up.png"))319        #expect(r.html.contains("href=\"https://example.org/keep\"") && r.html.contains("src=\"pic.png\"") && r.html.contains("href=\"local.css\"") && r.html.contains("url(img/ok.png)") && r.html.contains("data:image/png;base64,AAAA") && r.html.contains("color: red"))320        let relaxed = HTMLSanitizer.neutralize("<img src=\"../images/a.png\"><img src=\"//x.example/a.png\">", confine: false)321        #expect(relaxed.html.contains("../images/a.png") && relaxed.remote == ["//x.example/a.png"])322        #expect(HTMLSanitizer.classify("C:\\\\Windows\\\\x.png", confine: true) == .outside && HTMLSanitizer.classify("a/b/../c.png", confine: true) == .keep && HTMLSanitizer.classify("#frag", confine: true) == .keep)323    }324}325
Files main
Menu

Theme