SPB Git

Repository Public License

SafetyTests.swift 40.9 KB 608 lines swift

1// ORIX Convert — Packages/EngineDocuments/Tests/EngineDocumentsTests/SafetyTests.swift2// Description: Untrusted-input behaviour: macros never run, remote resources are never requested (local counting3//              listeners), parallel LibreOffice conversions, cancellation, time limit, damaged and hostile inputs.4// Author: Simon-Pierre Boucher — contact@spboucher.ai5import Testing6import Foundation7import Network8import PDFKit9@testable import EngineDocuments10import ConvertCore1112/// A TCP listener on 127.0.0.1 that counts every connection it receives and answers like a tiny HTTP server.13final class CountingListener: @unchecked Sendable {14    private let listener: NWListener15    private let lock = NSLock()16    private var _count = 017    private(set) var port: Int = 018    var count: Int { lock.lock(); defer { lock.unlock() }; return _count }1920    init() async throws {21        let parameters = NWParameters.tcp22        parameters.requiredLocalEndpoint = NWEndpoint.hostPort(host: "127.0.0.1", port: .any)23        listener = try NWListener(using: parameters)24        listener.newConnectionHandler = { [weak self] connection in25            self?.lock.lock(); self?._count += 1; self?.lock.unlock()26            connection.start(queue: .global())27            connection.receive(minimumIncompleteLength: 1, maximumLength: 65_536) { _, _, _, _ in28                let body = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok"29                connection.send(content: Data(body.utf8), completion: .contentProcessed { _ in connection.cancel() })30            }31        }32        try await withCheckedThrowingContinuation { (c: CheckedContinuation<Void, Error>) in33            let resumed = Flag()34            listener.stateUpdateHandler = { state in35                switch state {36                case .ready: if resumed.setOnce() { c.resume() }37                case .failed(let error): if resumed.setOnce() { c.resume(throwing: error) }38                default: break39                }40            }41            listener.start(queue: .global())42        }43        port = Int(listener.port?.rawValue ?? 0)44    }4546    /// Opens one connection to the listener (positive control that connections are counted).47    func poke() async {48        let connection = NWConnection(host: "127.0.0.1", port: NWEndpoint.Port(rawValue: UInt16(port))!, using: .tcp)49        await withCheckedContinuation { (c: CheckedContinuation<Void, Never>) in50            let resumed = Flag()51            connection.stateUpdateHandler = { state in52                switch state {53                case .ready:54                    connection.send(content: Data("GET /control HTTP/1.0\r\n\r\n".utf8), completion: .contentProcessed { _ in55                        connection.receive(minimumIncompleteLength: 1, maximumLength: 1024) { _, _, _, _ in connection.cancel(); if resumed.setOnce() { c.resume() } }56                    })57                case .failed, .cancelled: if resumed.setOnce() { c.resume() }58                default: break59                }60            }61            connection.start(queue: .global())62        }63    }6465    func stop() { listener.cancel() }6667    final class Flag: @unchecked Sendable {68        private let lock = NSLock(); private var done = false69        func setOnce() -> Bool { lock.lock(); defer { lock.unlock() }; if done { return false }; done = true; return true }70    }71}7273@Suite("Safety", .serialized)74struct SafetyTests {75    // MARK: Macros7677    /// An ODT assembled by hand with a Basic macro bound to the document's load event; the macro writes `marker`.78    static func autoOpenMacroODT(marker: URL, in dir: URL) throws -> URL {79        let module = """80        Sub Main81          Dim f As Integer82          f = FreeFile83          Open "\(marker.path)" For Output As #f84          Print #f, "macro ran"85          Close #f86        End Sub87        """88        var zip = ZipWriter()89        try zip.add("mimetype", Data("application/vnd.oasis.opendocument.text".utf8), store: true)90        try zip.add("content.xml", Data("""91        <?xml version="1.0" encoding="UTF-8"?>92        <office:document-content xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" xmlns:script="urn:oasis:names:tc:opendocument:xmlns:script:1.0" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:dom="http://www.w3.org/2001/xml-events" xmlns:ooo="http://openoffice.org/2004/office" office:version="1.3"><office:scripts><office:event-listeners><script:event-listener script:language="ooo:script" script:event-name="dom:load" xlink:href="vnd.sun.star.script:Standard.Module1.Main?language=Basic&amp;location=document" xlink:type="simple"/></office:event-listeners></office:scripts><office:body><office:text><text:p>Macro document body gerbil.</text:p></office:text></office:body></office:document-content>93        """.utf8))94        try zip.add("styles.xml", Data("<?xml version=\"1.0\" encoding=\"UTF-8\"?><office:document-styles xmlns:office=\"urn:oasis:names:tc:opendocument:xmlns:office:1.0\" xmlns:style=\"urn:oasis:names:tc:opendocument:xmlns:style:1.0\" office:version=\"1.3\"><office:styles><style:style style:name=\"Standard\" style:family=\"paragraph\"/></office:styles></office:document-styles>".utf8))95        try zip.add("Basic/script-lc.xml", Data("<?xml version=\"1.0\" encoding=\"UTF-8\"?><!DOCTYPE library:libraries PUBLIC \"-//OpenOffice.org//DTD OfficeDocument 1.0//EN\" \"libraries.dtd\"><library:libraries xmlns:library=\"http://openoffice.org/2000/library\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><library:library library:name=\"Standard\" library:link=\"false\"/></library:libraries>".utf8))96        try zip.add("Basic/Standard/script-lb.xml", Data("<?xml version=\"1.0\" encoding=\"UTF-8\"?><!DOCTYPE library:library PUBLIC \"-//OpenOffice.org//DTD OfficeDocument 1.0//EN\" \"library.dtd\"><library:library xmlns:library=\"http://openoffice.org/2000/library\" library:name=\"Standard\" library:readonly=\"false\" library:passwordprotected=\"false\"><library:element library:name=\"Module1\"/></library:library>".utf8))97        try zip.add("Basic/Standard/Module1.xml", Data("<?xml version=\"1.0\" encoding=\"UTF-8\"?><!DOCTYPE script:module PUBLIC \"-//OpenOffice.org//DTD OfficeDocument 1.0//EN\" \"module.dtd\"><script:module xmlns:script=\"http://openoffice.org/2000/script\" script:name=\"Module1\" script:language=\"StarBasic\">\(xmlEscape(module))</script:module>".utf8))98        try zip.add("META-INF/manifest.xml", Data("""99        <?xml version="1.0" encoding="UTF-8"?>100        <manifest:manifest xmlns:manifest="urn:oasis:names:tc:opendocument:xmlns:manifest:1.0" manifest:version="1.3"><manifest:file-entry manifest:full-path="/" manifest:version="1.3" manifest:media-type="application/vnd.oasis.opendocument.text"/><manifest:file-entry manifest:full-path="content.xml" manifest:media-type="text/xml"/><manifest:file-entry manifest:full-path="styles.xml" manifest:media-type="text/xml"/><manifest:file-entry manifest:full-path="Basic/Standard/Module1.xml" manifest:media-type="text/xml"/><manifest:file-entry manifest:full-path="Basic/Standard/script-lb.xml" manifest:media-type="text/xml"/><manifest:file-entry manifest:full-path="Basic/script-lc.xml" manifest:media-type="text/xml"/></manifest:manifest>101        """.utf8))102        let url = dir.appendingPathComponent("auto-open-macro.odt")103        try zip.finish().write(to: url)104        return url105    }106107    /// Positive control: the official, unslimmed LibreOffice with macro security switched off runs the macro when it108    /// opens the document. Returns `nil` when the official bundle is not on this machine (EmbeddedEngines/libreoffice/work).109    static func controlRunsMacro(_ document: URL, marker: URL, in dir: URL) async throws -> Bool? {110        guard let tools = BundledTools.searchDirectories().last else { return nil }111        let soffice = tools.deletingLastPathComponent().appendingPathComponent("libreoffice/work/LibreOffice.orig.app/Contents/MacOS/soffice")112        guard FileManager.default.isExecutableFile(atPath: soffice.path) else { return nil }113        let profile = dir.appendingPathComponent("control-profile")114        try FileManager.default.createDirectory(at: profile.appendingPathComponent("user"), withIntermediateDirectories: true)115        try Data("""116        <?xml version="1.0" encoding="UTF-8"?>117        <oor:items xmlns:oor="http://openoffice.org/2001/registry" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">118        <item oor:path="/org.openoffice.Office.Common/Security/Scripting"><prop oor:name="MacroSecurityLevel" oor:op="fuse"><value>0</value></prop></item>119        </oor:items>120        """.utf8).write(to: profile.appendingPathComponent("user/registrymodifications.xcu"))121        let process = Process()122        process.executableURL = soffice123        process.arguments = ["--headless", "--invisible", "--norestore", "--nologo", "--nodefault", "--nolockcheck", "-env:UserInstallation=" + profile.absoluteString.trimmingSuffix("/"), document.path]124        process.standardOutput = FileHandle.nullDevice; process.standardError = FileHandle.nullDevice125        try process.run()126        defer { if process.isRunning { process.terminate() } }127        for _ in 0..<60 {128            if FileManager.default.fileExists(atPath: marker.path) { return true }129            try await Task.sleep(nanoseconds: 250_000_000)130        }131        return false132    }133134    @Test func autoOpenMacroIsNeverExecuted() async throws {135        let dir = try Fixtures.newDirectory("macro")136        let marker = dir.appendingPathComponent("marker.txt")137        let document = try Self.autoOpenMacroODT(marker: marker, in: dir)138        // The fixture really is an auto-running macro document.139        if let ran = try await Self.controlRunsMacro(document, marker: marker, in: dir) {140            #expect(ran, "control: the official LibreOffice with macro security off runs the macro on open")141            try? FileManager.default.removeItem(at: marker)142        }143        #expect(!FileManager.default.fileExists(atPath: marker.path))144        let inspection = try await Engines.documents.inspect(document, format: .odt)145        #expect(inspection.fields.contains { $0.key == "Macros" && $0.value.hasPrefix("Present") })146        for (d, engine) in [(FormatID.pdf, "auto"), (.docx, "auto"), (.txt, "libreoffice"), (.html, "libreoffice"), (.doc, "auto"), (.markdown, "auto"), (.rtf, "apple")] {147            let run = try await convert(document, .odt, d, options: ["engine": .string(engine)])148            let out = try facts(run.file, d)149            #expect(normalized(out.searchText).contains("gerbil"), "\(d) carries the text")150            #expect(run.recorder.has("macros-dropped"), "\(d): the user is told the macros are dropped")151            #expect(!FileManager.default.fileExists(atPath: marker.path), "the macro did not run while converting to \(d)")152            if d == .docx {153                let zip = try ZipReader(url: run.file)154                #expect(!zip.entries.contains { $0.name.lowercased().contains("vba") }, "no macro part in the output")155            }156        }157        // Give a stray process time to write the marker, then check once more.158        try await Task.sleep(nanoseconds: 1_500_000_000)159        #expect(!FileManager.default.fileExists(atPath: marker.path))160        #expect(LibreOffice.registryModifications.contains("DisableMacrosExecution") && LibreOffice.registryModifications.contains("MacroSecurityLevel"))161    }162163    // MARK: Remote resources164165    @Test func remoteResourcesAreNeverRequested() async throws {166        let target = try await CountingListener(), proxy = try await CountingListener()167        defer { target.stop(); proxy.stop() }168        // Point the "dead proxy" of Pandoc and LibreOffice at a counting listener: a tool that even *tries* is seen.169        setenv("ORIX_CONVERT_TEST_PROXY_PORT", String(proxy.port), 1)170        defer { unsetenv("ORIX_CONVERT_TEST_PROXY_PORT") }171        #expect(Pandoc.deadPort == proxy.port && LibreOffice.registryModifications.contains(">\(proxy.port)<"))172        let fx = try await FixtureStore.shared.get()173        let dir = try Fixtures.newDirectory("remote")174        let base = "http://127.0.0.1:\(target.port)"175176        let html = dir.appendingPathComponent("remote.html")177        try Data("""178        <!DOCTYPE html><html><head><meta charset="utf-8"><title>Remote Test</title>179        <link rel="stylesheet" href="\(base)/style.css"><script src="\(base)/script.js"></script>180        <style>@import url("\(base)/import.css"); body { background: url("\(base)/bg.png"); } @font-face { font-family: X; src: url("\(base)/font.woff2"); }</style></head>181        <body><h1>Remote heading lemur</h1><p style="font-family: X">Text with a remote image <img src="\(base)/image.png" alt="remote"> and a local one <img src="pixel.png" alt="local">.</p>182        <iframe src="\(base)/frame.html"></iframe><video src="\(base)/video.mp4" poster="\(base)/poster.png"></video><object data="\(base)/object.svg"></object>183        <script>fetch("\(base)/fetch"); new Image().src = "\(base)/js-image.png";</script></body></html>184        """.utf8).write(to: html)185        try FileManager.default.copyItem(at: fx.pixel, to: dir.appendingPathComponent("pixel.png"))186        let markdown = dir.appendingPathComponent("remote.md")187        try Data("# Remote heading lemur\n\nRemote ![remote](\(base)/md-image.png) and local ![local](pixel.png).\n\n<img src=\"\(base)/raw-html.png\">\n".utf8).write(to: markdown)188        // DOCX whose picture is a link to the remote host instead of an embedded part.189        let docx = dir.appendingPathComponent("remote.docx")190        try repackage(fx.documents[.docx]!, to: docx, edits: [:], transform: [191            "word/document.xml": { $0.replacingOccurrences(of: #"r:embed="(rId\d+)""#, with: "r:link=\"rIdRemote\"", options: .regularExpression) },192            "word/_rels/document.xml.rels": { $0.replacingOccurrences(of: "</Relationships>", with: "<Relationship Id=\"rIdRemote\" Type=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships/image\" Target=\"\(base)/docx-image.png\" TargetMode=\"External\"/></Relationships>") },193        ])194        let docxFacts = try facts(docx, .docx)195        #expect(docxFacts.remoteReferences.contains("\(base)/docx-image.png"))196        // ODT whose picture is linked.197        let odt = dir.appendingPathComponent("remote.odt")198        try repackage(fx.documents[.odt]!, to: odt, edits: [:], transform: ["content.xml": { $0.replacingOccurrences(of: #"xlink:href="Pictures/[^"]+""#, with: "xlink:href=\"\(base)/odt-image.png\"", options: .regularExpression) }])199        let odtFacts = try facts(odt, .odt)200        #expect(odtFacts.remoteReferences == ["\(base)/odt-image.png"])201        // EPUB with a remote image in a chapter.202        let epub = dir.appendingPathComponent("remote.epub")203        let chapter = try #require(try EPUBPackage(url: fx.documents[.epub]!).spine.first { item in (try? ZipReader(url: fx.documents[.epub]!).data(for: item.path)).map { String(decoding: $0, as: UTF8.self).contains("<img") } ?? false })204        try repackage(fx.documents[.epub]!, to: epub, edits: [:], transform: [chapter.path: { $0.replacingOccurrences(of: #"<img src="[^"]+""#, with: "<img src=\"\(base)/epub-image.png\"", options: .regularExpression) }])205206        var cases: [(URL, FormatID, FormatID, String)] = [207            (html, .html, .pdf, "auto"), (html, .html, .docx, "auto"), (html, .html, .odt, "auto"), (html, .html, .epub, "auto"),208            (html, .html, .markdown, "auto"), (html, .html, .doc, "auto"), (html, .html, .rtf, "auto"), (html, .html, .txt, "auto"),209            (markdown, .markdown, .pdf, "auto"), (markdown, .markdown, .epub, "auto"), (markdown, .markdown, .docx, "auto"), (markdown, .markdown, .html, "auto"), (markdown, .markdown, .odt, "auto"),210            (docx, .docx, .pdf, "auto"), (docx, .docx, .odt, "libreoffice"), (docx, .docx, .doc, "libreoffice"), (docx, .docx, .html, "pandoc"), (docx, .docx, .html, "libreoffice"),211            (docx, .docx, .epub, "auto"), (docx, .docx, .rtf, "apple"),212            (odt, .odt, .pdf, "auto"), (odt, .odt, .docx, "libreoffice"), (odt, .odt, .html, "pandoc"), (odt, .odt, .html, "libreoffice"), (odt, .odt, .docx, "apple"),213            (epub, .epub, .pdf, "auto"), (epub, .epub, .html, "auto"), (epub, .epub, .docx, "auto"),214        ]215        cases.append((epub, .epub, .folder, Operations.epubChapters))216        for (source, s, d, engine) in cases {217            let run: Run218            do {219                run = engine == Operations.epubChapters220                    ? try await convert(source, s, d, operation: engine)221                    : try await convert(source, s, d, options: ["engine": .string(engine)])222            } catch {223                Issue.record("\(s)>\(d) [\(engine)] failed: \(String(describing: error).prefix(300))")224                continue225            }226            #expect(target.count == 0, "\(s)>\(d) [\(engine)] contacted the remote host")227            #expect(proxy.count == 0, "\(s)>\(d) [\(engine)] tried to go through the proxy (count \(proxy.count))")228            if s == .html || s == .markdown || s == .epub {229                #expect(run.recorder.has("remote-content-blocked"), "\(s)>\(d) [\(engine)] tells the user about the blocked content: \(run.recorder.warnings.map(\.code))")230            }231            if d != .folder, d != .doc {232                let out = try facts(run.file, d)233                #expect(out.wordCount > 3, "\(s)>\(d) [\(engine)] still converts the text")234                #expect(out.remoteReferences.isEmpty || d == .html || d == .markdown || engine != "auto" || s == .docx || s == .odt, "remote reference left in \(d): \(out.remoteReferences)")235            }236        }237        // HTML → PDF: the local image next to the source still loads, through the confined scheme handler.238        let pdf = try await convert(html, .html, .pdf)239        let pdfFacts = try facts(pdf.file, .pdf)240        #expect(pdfFacts.imageCount == 1, "exactly the local image is in the PDF: \(pdfFacts.imageCount)")241        #expect(normalized(pdfFacts.text).contains("lemur") && pdfFacts.title == "Remote Test")242        #expect(pdf.recorder.has("scripts-not-run") || pdf.recorder.has("remote-content-blocked"))243        // The Markdown → PDF route embeds the local image too.244        let mdPDF = try await convert(markdown, .markdown, .pdf)245        let mdPDFFacts = try facts(mdPDF.file, .pdf)246        #expect(mdPDFFacts.imageCount == 1)247        // Wait for stragglers, then prove the listeners were alive all along.248        try await Task.sleep(nanoseconds: 1_000_000_000)249        #expect(target.count == 0 && proxy.count == 0)250        await target.poke(); await proxy.poke()251        #expect(target.count == 1 && proxy.count == 1, "control: the listeners do count connections")252    }253254    // Regression (2026-11-21): a document whose only image is remote could not be converted at all. The image was255    // blocked and reported, then the engine's own validation failed with "Images kept: source 1, output 0".256    @Test func documentsWhoseOnlyImagesAreRemoteStillConvert() async throws {257        let target = try await CountingListener(), proxy = try await CountingListener()258        defer { target.stop(); proxy.stop() }259        setenv("ORIX_CONVERT_TEST_PROXY_PORT", String(proxy.port), 1)260        defer { unsetenv("ORIX_CONVERT_TEST_PROXY_PORT") }261        let fx = try await FixtureStore.shared.get()262        let dir = try Fixtures.newDirectory("remote-only")263        let base = "http://127.0.0.1:\(target.port)"264265        let html = dir.appendingPathComponent("only.html")266        try Data("<!DOCTYPE html><html><head><meta charset=\"utf-8\"><title>Remote Only</title></head><body><h1>Remote heading lemur</h1><p>Paragraph with a remote image <img src=\"\(base)/a.png\" alt=\"a\"> and a second <img src=\"\(base)/b.png\" alt=\"b\"> gibbon.</p></body></html>".utf8).write(to: html)267        let markdown = dir.appendingPathComponent("only.md")268        try Data("# Remote heading lemur\n\nParagraph gibbon with a remote image ![remote](\(base)/md.png) inside.\n".utf8).write(to: markdown)269        let latex = dir.appendingPathComponent("only.tex")270        try Data("\\documentclass{article}\n\\usepackage{graphicx}\n\\begin{document}\n\\section{Remote heading lemur}\nParagraph gibbon with a remote image.\n\n\\includegraphics[width=3cm]{\(base)/tex.png}\n\\end{document}\n".utf8).write(to: latex)271        let rst = dir.appendingPathComponent("only.rst")272        try Data("Remote heading lemur\n====================\n\nParagraph gibbon with a remote image.\n\n.. image:: \(base)/rst.png\n   :alt: remote\n".utf8).write(to: rst)273        // EPUB whose every image is remote.274        let epub = dir.appendingPathComponent("only.epub")275        var transforms: [String: (String) -> String] = [:]276        for item in try EPUBPackage(url: fx.documents[.epub]!).spine {277            transforms[item.path] = { $0.replacingOccurrences(of: #"<img src="[^"]+""#, with: "<img src=\"\(base)/epub.png\"", options: .regularExpression) }278        }279        try repackage(fx.documents[.epub]!, to: epub, edits: [:], transform: transforms)280281        // The facts tell remote images from local ones.282        #expect(try facts(html, .html).imageCount == 2 && facts(html, .html).remoteImageCount == 2)283        #expect(try facts(markdown, .markdown).remoteImageCount == 1 && facts(latex, .latex).remoteImageCount == 1 && facts(rst, .rst).remoteImageCount == 1)284        let epubFacts = try facts(epub, .epub)285        #expect(epubFacts.imageCount >= 1 && epubFacts.remoteImageCount == epubFacts.imageCount && epubFacts.deliverableImageCount == 0)286        #expect(try facts(fx.documents[.html]!, .html).remoteImageCount == 0, "a local image is not counted as remote")287288        let cases: [(URL, FormatID, FormatID)] = [289            (html, .html, .pdf), (html, .html, .docx), (html, .html, .odt), (html, .html, .rtf), (html, .html, .epub),290            (markdown, .markdown, .pdf), (markdown, .markdown, .docx), (markdown, .markdown, .epub), (markdown, .markdown, .html), (markdown, .markdown, .odt),291            (latex, .latex, .docx), (latex, .latex, .html),292            (rst, .rst, .html), (rst, .rst, .docx), (rst, .rst, .pdf),293            (epub, .epub, .pdf), (epub, .epub, .docx), (epub, .epub, .html),294        ]295        for (source, s, d) in cases {296            let run: Run297            do { run = try await convert(source, s, d) } catch {298                Issue.record("\(s)>\(d) failed: \(String(describing: error).prefix(300))")299                continue300            }301            #expect(run.failedChecks.isEmpty, "\(s)>\(d): \(run.failedChecks)")302            #expect(target.count == 0, "\(s)>\(d) contacted the remote host")303            #expect(proxy.count == 0, "\(s)>\(d) tried to go through the proxy")304            // One warning, saying how many images are left out and why.305            let blocked = run.recorder.warnings.filter { $0.code == "remote-content-blocked" }306            #expect(blocked.count == 1, "\(s)>\(d): \(run.recorder.warnings.map(\.code))")307            let expectedImages = s == .html ? "2 images are not included" : (s == .epub ? "not included because" : "1 image is not included")308            #expect(blocked.first?.message.contains(expectedImages) == true && blocked.first?.message.contains("remote server") == true && blocked.first?.severity == .dataLoss,309                    "\(s)>\(d): \(blocked.first?.message ?? "no warning")")310            // The remote images are not reported a second time as dropped local images.311            #expect(!run.recorder.has("images-dropped"), "\(s)>\(d): \(run.recorder.warnings.filter { $0.code == "images-dropped" }.map(\.message))")312            if let check = run.checks.first(where: { $0.name == "Images kept" }) { #expect(check.passed && check.detail.contains("remote image(s) not downloaded")) }313            let out = try facts(run.file, d)314            #expect(normalized(out.searchText).contains("gibbon") || s == .epub, "\(s)>\(d) still converts the text")315            #expect(out.wordCount > 3)316            if d == .pdf || d == .docx || d == .odt, s != .epub { #expect(out.imageCount == 0, "\(s)>\(d): no image can be in the result, found \(out.imageCount)") }317            // Preflight says it before converting.318            let pre = Engines.all.first { $0.path(for: run.request) != nil }?.preflight(run.request, inspection: nil) ?? []319            #expect(pre.contains { $0.code == "remote-content-blocked" && $0.message.contains("will not be included") }, "\(s)>\(d) preflight: \(pre.map(\.message))")320        }321        // A document with a remote image AND a local one: the local one is still required in the output.322        let mixed = dir.appendingPathComponent("mixed.md")323        try FileManager.default.copyItem(at: fx.pixel, to: dir.appendingPathComponent("pixel.png"))324        try Data("# Mixed lemur\n\nRemote ![r](\(base)/m.png) and local ![l](pixel.png) gibbon.\n".utf8).write(to: mixed)325        let mixedRun = try await convert(mixed, .markdown, .docx)326        let kept = try #require(mixedRun.checks.first { $0.name == "Images kept" })327        #expect(kept.passed && kept.detail.hasPrefix("source 1, output 1"), "\(kept.detail)")328        #expect(mixedRun.output.facts["images.remote-not-included"] == "1")329        try await Task.sleep(nanoseconds: 1_000_000_000)330        #expect(target.count == 0 && proxy.count == 0)331        await target.poke(); await proxy.poke()332        #expect(target.count == 1 && proxy.count == 1, "control: the listeners do count connections")333    }334335    @Test func markupIncludesStayInsideTheFolder() async throws {336        let target = try await CountingListener(), proxy = try await CountingListener()337        defer { target.stop(); proxy.stop() }338        setenv("ORIX_CONVERT_TEST_PROXY_PORT", String(proxy.port), 1)339        defer { unsetenv("ORIX_CONVERT_TEST_PROXY_PORT") }340        let dir = try Fixtures.newDirectory("includes")341        let inner = dir.appendingPathComponent("project", isDirectory: true)342        try FileManager.default.createDirectory(at: inner, withIntermediateDirectories: true)343        try Data("SECRETOUTSIDE content that must never be included.\n".utf8).write(to: dir.appendingPathComponent("secret.txt"))344        try Data("Included paragraph tamarin.\n".utf8).write(to: inner.appendingPathComponent("part.rst"))345        try Data("Included paragraph tamarin.\n".utf8).write(to: inner.appendingPathComponent("part.tex"))346        let base = "http://127.0.0.1:\(target.port)"347        let rst = inner.appendingPathComponent("main.rst")348        try Data("""349        Main Title350        ==========351352        .. include:: part.rst353354        .. include:: \(dir.path)/secret.txt355356        .. include:: ../secret.txt357358        .. include:: \(base)/remote.rst359360        .. literalinclude:: /etc/shells361362        .. csv-table:: Remote table363           :url: \(base)/table.csv364365        .. csv-table:: Outside table366           :file: /etc/shells367368        .. image:: \(base)/image.png369370        Closing paragraph okapi.371372        """.utf8).write(to: rst)373        let tex = inner.appendingPathComponent("main.tex")374        try Data("""375        \\documentclass{article}376        \\begin{document}377        Opening paragraph.378379        \\input{part}380381        \\input{\(dir.path)/secret.txt}382        \\input{../secret.txt}383        \\include{/etc/shells}384        \\lstinputlisting{/etc/shells}385        \\verbatiminput{\(dir.path)/secret.txt}386        \\input{\(base)/remote.tex}387        \\includegraphics{\(base)/image.png}388389        Closing paragraph okapi.390        \\end{document}391392        """.utf8).write(to: tex)393        for (source, s) in [(rst, FormatID.rst), (tex, .latex)] {394            for d in [FormatID.html, .docx, .markdown, .pdf] where !(s == .latex && d == .pdf) {395                let run = try await convert(source, s, d)396                let text = normalized(try facts(run.file, d).searchText)397                #expect(text.contains("tamarin"), "\(s)>\(d): the include inside the folder still works")398                #expect(text.contains("okapi"), "\(s)>\(d): the document converts to the end")399                #expect(!text.contains("secretoutside") && !text.contains("/bin/zsh"), "\(s)>\(d): nothing outside the folder was read")400                #expect(run.recorder.has("local-resource-refused") && run.recorder.has("remote-content-blocked"), "\(s)>\(d): \(run.recorder.warnings.map(\.code))")401                #expect(target.count == 0 && proxy.count == 0, "\(s)>\(d): no network attempt (target \(target.count), proxy \(proxy.count))")402            }403        }404        await target.poke(); await proxy.poke()405        #expect(target.count == 1 && proxy.count == 1)406    }407408    @Test func htmlCannotReadOutsideItsFolder() async throws {409        let dir = try Fixtures.newDirectory("confine")410        let inner = dir.appendingPathComponent("site", isDirectory: true)411        try FileManager.default.createDirectory(at: inner, withIntermediateDirectories: true)412        try Fixtures.writePNG(dir.appendingPathComponent("outside.png"))413        try Fixtures.writePNG(inner.appendingPathComponent("inside.png"))414        try FileManager.default.createSymbolicLink(at: inner.appendingPathComponent("sneaky.png"), withDestinationURL: dir.appendingPathComponent("outside.png"))415        let html = inner.appendingPathComponent("page.html")416        try Data("""417        <html><body><p>Confinement okapi</p><img src="inside.png"><img src="../outside.png"><img src="sneaky.png">418        <img src="file://\(dir.path)/outside.png"><img src="/etc/hosts"><iframe src="file:///etc/passwd"></iframe></body></html>419        """.utf8).write(to: html)420        let run = try await convert(html, .html, .pdf)421        #expect(run.failedChecks.isEmpty, "\(run.failedChecks)")422        let f = try facts(run.file, .pdf)423        // Ask the renderer exactly what it served: one file, from inside the folder; the symlink that points out of424        // the folder is refused; "../", absolute and file: references never reach a file outside it.425        let direct = try await WebPDFRenderer.render(html: try Data(contentsOf: html), mainName: "page.html", baseDirectory: inner, setup: PageSetup(), output: dir.appendingPathComponent("direct.pdf"))426        #expect(direct.localResourcesServed == 1, "served: \(direct.localResourcesServed)")427        #expect(direct.refusedOutsideFolder == ["/sneaky.png"], "refused: \(direct.refusedOutsideFolder)")428        #expect(Set(direct.localResourcesMissing) == ["/outside.png", "/etc/hosts"], "not found inside the folder: \(direct.localResourcesMissing)")429        #expect(f.imageCount >= 1)430        #expect(!f.text.contains("localhost") && !f.text.contains("root:"), "no system file leaked into the PDF")431        #expect(run.recorder.has("local-resource-refused") || run.recorder.has("local-resource-missing"))432    }433434    // MARK: Concurrency, cancellation, time limit435436    @Test func eightParallelLibreOfficeConversions() async throws {437        let fx = try await FixtureStore.shared.get()438        let source = fx.documents[.docx]!439        let started = Date()440        let results = await withTaskGroup(of: Bool.self) { group -> [Bool] in441            for i in 0..<8 {442                group.addTask {443                    guard let run = try? await convert(source, .docx, i % 2 == 0 ? .pdf : .odt) else { return false }444                    return run.failedChecks.isEmpty445                }446            }447            var all: [Bool] = []448            for await ok in group { all.append(ok) }449            return all450        }451        #expect(results.count == 8 && results.allSatisfy { $0 }, "all eight succeed: \(results)")452        print("PERF eight parallel LibreOffice conversions: \(String(format: "%.2f", Date().timeIntervalSince(started))) s")453    }454455    static func sofficeProcesses(mentioning path: String) -> Int {456        let p = Process()457        p.executableURL = URL(fileURLWithPath: "/usr/bin/pgrep")458        p.arguments = ["-f", path]459        let pipe = Pipe(); p.standardOutput = pipe; p.standardError = FileHandle.nullDevice460        guard (try? p.run()) != nil else { return -1 }461        p.waitUntilExit()462        return String(decoding: pipe.fileHandleForReading.readDataToEndOfFile(), as: UTF8.self).split(separator: "\n").count463    }464465    @Test func cancellationStopsLibreOffice() async throws {466        let fx = try await FixtureStore.shared.get()467        let dir = try Fixtures.newDirectory("cancel")468        let request = ConversionRequest(source: fx.documents[.docx]!, sourceFormat: .docx, destinationFormat: .pdf)469        let task = Task { try await Engines.documents.convert(request, context: ConversionContext(workDirectory: dir)) }470        try await Task.sleep(nanoseconds: 350_000_000)471        let started = Date()472        task.cancel()473        do {474            _ = try await task.value475            Issue.record("a cancelled conversion must not return a result")476        } catch is CancellationError {477        } catch { Issue.record("expected CancellationError, got \(error)") }478        #expect(Date().timeIntervalSince(started) < 10, "cancellation returns promptly")479        try await Task.sleep(nanoseconds: 800_000_000)480        #expect(Self.sofficeProcesses(mentioning: dir.path) == 0, "no LibreOffice process of the cancelled job is left running")481        let leftovers = try FileManager.default.contentsOfDirectory(atPath: dir.path)482        #expect(leftovers.isEmpty, "the work directory is clean after cancellation: \(leftovers)")483        // A task cancelled before it starts never launches anything.484        let early = Task { try await Engines.presentations.convert(ConversionRequest(source: fx.deck[.pptx]!, sourceFormat: .pptx, destinationFormat: .pdf), context: ConversionContext(workDirectory: dir)) }485        early.cancel()486        await #expect(throws: (any Error).self) { _ = try await early.value }487    }488489    @Test func timeLimitStopsLibreOffice() async throws {490        let fx = try await FixtureStore.shared.get()491        let dir = try Fixtures.newDirectory("timeout")492        let started = Date()493        do {494            _ = try await LibreOffice.convert(fx.documents[.docx]!, filter: "pdf:writer_pdf_Export", scratch: dir, timeout: 0.15)495            Issue.record("a 0.15 s limit cannot be met")496        } catch let error as ConversionError {497            guard case .limitExceeded(let message) = error else { Issue.record("wrong error \(error)"); return }498            #expect(message.contains("LibreOffice"))499        }500        #expect(Date().timeIntervalSince(started) < 10)501        try await Task.sleep(nanoseconds: 800_000_000)502        #expect(Self.sofficeProcesses(mentioning: dir.path) == 0, "the timed-out LibreOffice was killed")503        // The option is clamped to a sane range and exposed on every path.504        let path = try #require(Engines.path("documents:docx>pdf"))505        #expect(Engines.documents.options(for: path).contains { $0.id == "timeout" })506    }507508    // MARK: Damaged and hostile inputs509510    @Test func damagedInputsFailCleanly() async throws {511        let fx = try await FixtureStore.shared.get()512        let dir = try Fixtures.newDirectory("damaged")513        func truncated(_ format: FormatID, _ source: URL, keep: Double) throws -> URL {514            let data = try Data(contentsOf: source)515            let url = dir.appendingPathComponent("truncated-\(Int(keep * 100)).\(source.pathExtension)")516            try data.prefix(Int(Double(data.count) * keep)).write(to: url)517            return url518        }519        var cases: [(URL, FormatID, FormatID, String?)] = []520        for (format, source) in [(FormatID.docx, fx.documents[.docx]!), (.odt, fx.documents[.odt]!), (.epub, fx.documents[.epub]!)] {521            cases.append((try truncated(format, source, keep: 0.5), format, .pdf, nil))522            cases.append((try truncated(format, source, keep: 0.5), format, .markdown, nil))523        }524        cases.append((try truncated(.pptx, fx.deck[.pptx]!, keep: 0.6), .pptx, .pdf, nil))525        cases.append((try truncated(.odp, fx.deck[.odp]!, keep: 0.6), .odp, .png, nil))526        cases.append((try truncated(.epub, fx.documents[.epub]!, keep: 0.4), .epub, .folder, Operations.epubChapters))527        let garbage = Data((0..<4096).map { UInt8(truncatingIfNeeded: $0 &* 37 &+ 11) })528        for (ext, format, d) in [("doc", FormatID.doc, FormatID.pdf), ("rtf", .rtf, .docx), ("ppt", .ppt, .pdf), ("docx", .docx, .txt), ("epub", .epub, .html), ("odp", .odp, .pdf)] {529            let url = dir.appendingPathComponent("garbage.\(ext)")530            try garbage.write(to: url)531            cases.append((url, format, d, nil))532        }533        // A DOCX whose main part is not well-formed XML, and one whose main part is missing.534        let badXML = dir.appendingPathComponent("bad-xml.docx")535        try repackage(fx.documents[.docx]!, to: badXML, edits: ["word/document.xml": Data("<w:document><w:body><w:p>unclosed".utf8)])536        cases.append((badXML, .docx, .pdf, nil))537        let noMain = dir.appendingPathComponent("no-main.docx")538        let removal: [String: Data?] = ["word/document.xml": nil]539        try repackage(fx.documents[.docx]!, to: noMain, edits: removal)540        cases.append((noMain, .docx, .odt, nil))541        // An empty file and a missing file.542        let empty = dir.appendingPathComponent("empty.odt"); try Data().write(to: empty)543        cases.append((empty, .odt, .pdf, nil))544        cases.append((dir.appendingPathComponent("missing.docx"), .docx, .pdf, nil))545        // An "encrypted" OOXML file (OLE2 container) and a text file that is not valid in the forced encoding.546        let ole = dir.appendingPathComponent("protected.docx")547        try (Data([0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]) + Data(count: 2048)).write(to: ole)548        cases.append((ole, .docx, .pdf, nil))549550        for (url, s, d, op) in cases {551            do {552                _ = try await convert(url, s, d, operation: op)553                Issue.record("\(url.lastPathComponent) → \(d) should fail")554            } catch let error as ConversionError {555                switch error {556                case .malformedInput, .unreadableInput, .encryptedInput, .limitExceeded: break557                default: Issue.record("\(url.lastPathComponent) → \(d): unexpected error kind \(error)")558                }559                #expect(error.errorDescription?.isEmpty == false)560            } catch { Issue.record("\(url.lastPathComponent) → \(d): non-ConversionError \(error)") }561            // Inspection and preflight never crash on the same file.562            let engine = Engines.all.first { $0.inspectableFormats.contains(s) }!563            _ = try? await engine.inspect(url, format: s)564            _ = engine.preflight(ConversionRequest(source: url, sourceFormat: s, destinationFormat: d, operation: op), inspection: nil)565        }566        do { _ = try await convert(ole, .docx, .pdf); Issue.record("must be refused") } catch let error as ConversionError {567            if case .encryptedInput = error {} else { Issue.record("OLE2-wrapped DOCX is reported as protected, got \(error)") }568        }569        let i = try await Engines.documents.inspect(ole, format: .docx)570        #expect(i.isEncrypted == true)571    }572573    @Test func zipReaderIsBounded() throws {574        var writer = ZipWriter()575        let big = Data(repeating: 0x41, count: 4 << 20)576        try writer.add("mimetype", Data("x".utf8), store: true)577        try writer.add("big.txt", big)578        try writer.add("dir/é name.txt", Data("accent".utf8))579        let archive = writer.finish()580        #expect(archive.count < 64 << 10, "4 MiB of one byte deflates to a few KiB (\(archive.count) bytes)")581        let zip = try ZipReader(data: archive)582        #expect(zip.entries.map(\.name) == ["mimetype", "big.txt", "dir/é name.txt"] && zip.entries[0].method == 0 && zip.entries[1].method == 8)583        let roundTrip = try zip.data(for: "big.txt")584        #expect(roundTrip == big)585        let accent = try zip.data(for: "dir/é name.txt")586        #expect(String(decoding: accent, as: UTF8.self) == "accent")587        // A decompression bomb stops at the limit instead of filling memory.588        #expect(throws: ConversionError.self) { _ = try zip.data(for: "big.txt", limit: 1 << 20) }589        // Corrupted payload: checksum mismatch or inflate failure, never wrong data.590        var corrupt = archive591        let offset = corrupt.count / 3592        corrupt[offset] ^= 0xFF; corrupt[offset + 1] ^= 0xFF593        #expect(throws: ConversionError.self) { _ = try ZipReader(data: corrupt).data(for: "big.txt") }594        #expect(throws: ConversionError.self) { _ = try ZipReader(data: archive.prefix(archive.count - 30)) }595        #expect(throws: ConversionError.self) { _ = try ZipReader(data: Data("not a zip".utf8)) }596        // Entry names never escape: EPUB paths are normalised below the package root.597        #expect(EPUBPackage.resolve("../../etc/passwd", base: "EPUB/text") == "etc/passwd")598        #expect(EPUBPackage.resolve("a/./b/../c%20d.xhtml#frag", base: "EPUB") == "EPUB/a/c d.xhtml")599        #expect(Operations.safeFileName("../../evil:name\n.txt", fallback: "x") == "-..-evil-name-.txt")600        // XML: external entities are not resolved and nesting is bounded.601        let xxe = Data("<?xml version=\"1.0\"?><!DOCTYPE r [<!ENTITY x SYSTEM \"file:///etc/hosts\">]><r>&x;</r>".utf8)602        let parsed = try? XMLTree.parse(xxe)603        #expect(parsed == nil || parsed?.deepText.contains("localhost") == false, "an external entity is never read")604        let deep = Data((String(repeating: "<a>", count: 2000) + String(repeating: "</a>", count: 2000)).utf8)605        #expect(throws: ConversionError.self) { _ = try XMLTree.parse(deep) }606    }607}608
Files main
Menu

Theme