SPB Git

Repository Public License

SecurityTests.swift 32.0 KB 414 lines swift

1// ORIX Convert — Packages/EngineArchive/Tests/EngineArchiveTests/SecurityTests.swift2// Description: Hostile archives crafted for the purpose: path traversal, link attacks, special files, decompression3//              bombs, limits, truncation and corruption, quarantine propagation, cancellation.4// Author: Simon-Pierre Boucher — contact@spboucher.ai5import CArchive6import ConvertCore7@testable import EngineArchive8import Foundation9import Testing1011@Suite struct SecurityTests {12    static let extract = ArchiveEngine.extractOperation1314    /// Everything under the scratch folder except the extraction's own work folders: must not change.15    static func outside(_ scratch: Scratch) throws -> Set<String> {16        Set(try Tree.snapshot(scratch.root).keys.filter { !$0.hasPrefix("work-") })17    }1819    /// Writes an archive through libarchive with names exactly as given — no sanitising — for formats that cannot20    /// be assembled by hand (7z).21    static func hostile(_ format: FormatID, names: [String], to url: URL) throws {22        try ArchiveLocale.withUTF8 {23            let writer = try ArchiveWriter(url: url, format: format, settings: WriteSettings(preset: .fast))24            for name in names {25                let entry = try #require(archive_entry_new())26                defer { archive_entry_free(entry) }27                archive_entry_set_pathname_utf8(entry, name)28                archive_entry_set_filetype(entry, UInt32(S_IFREG))29                archive_entry_set_perm(entry, 0o644)30                let data = Array("payload of \(name)".utf8)31                archive_entry_set_size(entry, Int64(data.count))32                try writer.writeHeader(entry)33                try data.withUnsafeBytes { try writer.write($0) }34                try writer.finishEntry()35            }36            try writer.close()37        }38    }3940    static let traversalNames = ["good/ok.txt", "../evil1.txt", "a/../../evil2.txt", "..\\..\\evil4.txt", "C:\\Windows\\evil3.txt", "./././dot.txt", "bad\u{1}name.txt",41                                 "photo\u{202E}gpj.exe", "foo/..namedfork/rsrc", "...", String(repeating: "a", count: 300) + ".txt",42                                 (0..<200).map { "d\($0)" }.joined(separator: "/") + "/deep.txt"]43    static let traversalExpected: Set<String> = ["good", "good/ok.txt", "Windows", "Windows/evil3.txt", "dot.txt", "bad_name.txt", "photo_gpj.exe", "_.."]4445    @Test(arguments: [FormatID.tar, .zip, .sevenZip, .tgz])46    func pathTraversalNeverLeavesTheDestination(format: FormatID) async throws {47        let scratch = try Scratch(); defer { scratch.cleanUp() }48        let absolute = "/tmp/orix-absolute-\(UUID().uuidString).txt"49        let names = Self.traversalNames + [absolute]50        let archive = scratch.url("hostile.\(Fixture.ext(format))")51        switch format {52        case .tar: try Craft.tar(names.map { Craft.TarEntry($0, data: "payload") }).write(to: archive)53        case .zip: try Craft.zip(names.map { Craft.ZipEntry($0, "payload") }).write(to: archive)54        default: try Self.hostile(format, names: names, to: archive)55        }56        let before = try Self.outside(scratch)57        let out = try await runEngine([archive], format, .folder, operation: Self.extract, name: "out", in: scratch)58        expectAllPass(out.checks, "traversal \(format)")59        let items = try Tree.snapshot(out.url)60        let absoluteRelative = String(absolute.dropFirst())61        #expect(Set(items.keys) == Self.traversalExpected.union(["tmp", absoluteRelative]), "\(format): \(items.keys.sorted())")62        // Nothing appeared anywhere else: not next to the output, not in the scratch folder, not at the absolute path.63        #expect(try FileManager.default.contentsOfDirectory(atPath: out.work.path) == ["out"])64        #expect(try Self.outside(scratch) == before)65        #expect(!FileManager.default.fileExists(atPath: absolute))66        #expect(out.collector.warnings.filter { $0.code == "entries-rejected" }.map(\.message).joined().contains("climbs out of the folder"))67        #expect(out.collector.warnings.filter { $0.code == "entries-rejected" }.count == 4)           // traversal, too long, too deep, resource fork68        #expect(out.collector.message("names-sanitized").hasPrefix("6 entries had an unsafe name"), "\(out.collector.message("names-sanitized"))")69        #expect(out.collector.warnings.allSatisfy { $0.severity != .info })70        // The listing flags the same names before anything is extracted.71        let page = try await ArchiveBrowser.list(archive, format: format)72        #expect(page.summary.unsafeNames == 12 && page.entries.first { $0.path == "good/ok.txt" }?.hasSafeName == true, "\(page.summary.unsafeNames) unsafe among \(page.entries.map(\.path))")73        let inspection = try await ArchiveEngine().inspect(archive, format: format)74        #expect(inspection.fields.contains { $0.key == "Entries with unsafe names" && $0.value == "12" })75        // Converting to another archive applies the same rules to the names it writes.76        let converted = try await runEngine([archive], format, format == .zip ? .tar : .zip, operation: nil, in: scratch)77        expectAllPass(converted.checks, "traversal convert \(format)")78        let names2 = try await ArchiveBrowser.list(converted.url, format: format == .zip ? .tar : .zip).entries.map(\.path)79        #expect(Set(names2) == ["good/ok.txt", "Windows/evil3.txt", "dot.txt", "bad_name.txt", "photo_gpj.exe", "_..", absoluteRelative])80    }8182    @Test func everyEntryRefusedIsAnErrorNotAnEmptyFolder() async throws {83        let scratch = try Scratch(); defer { scratch.cleanUp() }84        let archive = try scratch.file("only-evil.tar", Craft.tar([Craft.TarEntry("../a.txt", data: "x"), Craft.TarEntry("../../b.txt", data: "y")]))85        let work = try scratch.work()86        do {87            _ = try await ArchiveEngine().convert(.init(source: archive, sourceFormat: .tar, destinationFormat: .folder, operation: Self.extract, outputBaseName: "out"), context: .init(workDirectory: work))88            Issue.record("accepted an archive of traversal entries only")89        } catch ConversionError.outputInvalid(let message) { #expect(message.contains("refused for safety")) }90        #expect(try FileManager.default.contentsOfDirectory(atPath: work.path).isEmpty)91        #expect(try Self.outside(scratch) == ["only-evil.tar"])92    }9394    @Test func namesWithNulBytesNormalisationAndCaseCollisions() async throws {95        let scratch = try Scratch(); defer { scratch.cleanUp() }96        let entries = [Craft.ZipEntry(rawName: Array("nul".utf8) + [0] + Array("/../../escape.txt".utf8), data: "nul", utf8: true),97                       Craft.ZipEntry("caf\u{00E9}.txt", "precomposed"), Craft.ZipEntry("cafe\u{0301}.txt", "decomposed"),98                       Craft.ZipEntry("Case.txt", "upper"), Craft.ZipEntry("case.TXT", "lower"),99                       Craft.ZipEntry("thing", "a file"), Craft.ZipEntry("thing/inside.txt", "through a file"),100                       Craft.ZipEntry("folder/", "", unixMode: 0o040755), Craft.ZipEntry("folder", "replaces a folder?"),101                       Craft.ZipEntry(rawName: [0x66, 0x69, 0x63, 0x68, 0x69, 0x65, 0x72, 0x82, 0x2E, 0x74, 0x78, 0x74], data: "cp437")]      // "fichier" + CP437 é102        let archive = try scratch.file("names.zip", Craft.zip(entries))103        let before = try Self.outside(scratch)104        let out = try await runEngine([archive], .zip, .folder, operation: Self.extract, name: "out", in: scratch)105        expectAllPass(out.checks, "names")106        let items = try Tree.snapshot(out.url)107        // One file per colliding pair, holding the LAST entry's data; the NUL-truncated name stays inside.108        #expect(items.count == 6, "\(items.keys.sorted())")109        #expect(items.keys.contains("nul") && items.keys.contains("thing") && items.keys.contains("folder") && items["folder"]?.kind == "dir")110        #expect(try String(contentsOf: out.url.appendingPathComponent("caf\u{00E9}.txt"), encoding: .utf8) == "decomposed")111        #expect(try String(contentsOf: out.url.appendingPathComponent("case.txt"), encoding: .utf8) == "lower")112        #expect(try String(contentsOf: out.url.appendingPathComponent("thing"), encoding: .utf8) == "a file")113        #expect(try String(contentsOf: out.url.appendingPathComponent("fichier\u{00E9}.txt"), encoding: .utf8) == "cp437")114        #expect(out.collector.message("duplicates-replaced").hasPrefix("2 entries repeat"))115        #expect(out.collector.message("names-legacy-encoding").hasPrefix("1 name is not UTF-8"))116        let rejected = out.collector.warnings.filter { $0.code == "entries-rejected" }.map(\.message).joined(separator: " | ")117        #expect(rejected.contains("passes through a symbolic link or a file") && rejected.contains("already used by a folder or a link"))118        #expect(try Self.outside(scratch) == before)119    }120121    @Test func linkAttacksAreBlocked() async throws {122        let scratch = try Scratch(); defer { scratch.cleanUp() }123        let outsideDir = try scratch.directory("outside")124        let secret = try scratch.file("outside/secret.txt", Data("do not touch".utf8))125        let E = Craft.TarEntry.self126        let entries = [127            // 1. A link to an absolute folder outside, then a file "through" it.128            E.init("abs-link", type: "2", link: outsideDir.path), E.init("abs-link/pwned1.txt", data: "pwned"),129            // 2. A relative link that climbs out, then a file through it.130            E.init("sub/rel-link", type: "2", link: "../../outside"), E.init("sub/rel-link/pwned2.txt", data: "pwned"),131            // 3. A link that stays inside is allowed, but nothing is ever written through a link.132            E.init("realdir/", type: "5", mode: 0o755), E.init("inner", type: "2", link: "realdir"), E.init("inner/through.txt", data: "through a link"),133            // 4. Two links that are harmless one by one and escape together.134            E.init("x/y/z/d", type: "2", link: "../../.."), E.init("e", type: "2", link: "x/y/z/d/.."),135            // 5. A link made dangerous by a later entry.136            E.init("e2", type: "2", link: "q/.."), E.init("q", type: "2", link: "."),137            // 6. Hard links to files outside the archive, and one legitimate hard link.138            E.init("good.txt", data: "good"), E.init("hl-abs", type: "1", link: secret.path), E.init("hl-rel", type: "1", link: "../outside/secret.txt"),139            E.init("hl-missing", type: "1", link: "never-extracted.txt"), E.init("hl-good", type: "1", link: "good.txt"),140            // 7. Replacing a file by a link, and writing over a link.141            E.init("victim.txt", data: "victim"), E.init("victim.txt", type: "2", link: outsideDir.path + "/secret.txt"),142            E.init("s", type: "2", link: "victim.txt"), E.init("s", data: "overwritten through s"),143            // 8. A link to the secret by a long absolute path hidden in a relative-looking one.144            E.init("sneaky", type: "2", link: "realdir/../../outside/secret.txt"),145        ]146        let archive = try scratch.file("links.tar", Craft.tar(entries))147        let before = try Tree.snapshot(outsideDir)148        let out = try await runEngine([archive], .tar, .folder, operation: Self.extract, name: "out", in: scratch)149        expectAllPass(out.checks, "links")150        #expect(out.checks.contains { $0.name == "output.no-symlink-escape" && $0.passed })151        let items = try Tree.snapshot(out.url)152        // The outside folder is untouched: same names, same content, and no extra hard link to the secret.153        #expect(try Tree.snapshot(outsideDir) == before)154        var info = stat()155        #expect(lstat(secret.path, &info) == 0 && info.st_nlink == 1)156        // 1 and 2: the links were refused, so the "through" files landed in real folders inside the destination.157        #expect(items["abs-link"]?.kind == "dir" && items["abs-link/pwned1.txt"]?.kind == "file" && items["sub/rel-link"]?.kind == "dir" && items["sub/rel-link/pwned2.txt"]?.kind == "file")158        // 3: the inside link exists, the write through it did not happen.159        #expect(items["inner"]?.kind == "link" && items["inner"]?.target == "realdir" && items["realdir/through.txt"] == nil)160        // 4 and 5: the link that escapes once combined is gone; its harmless partner stays.161        #expect(items["x/y/z/d"]?.kind == "link" && items["e"] == nil && items["e2"] == nil && items["q"]?.kind == "link")162        // 6: only the legitimate hard link exists, and it is the same file.163        #expect(items["hl-abs"] == nil && items["hl-rel"] == nil && items["hl-missing"] == nil && items["hl-good"]?.kind == "file")164        var a = stat(), b = stat()165        #expect(lstat(out.url.appendingPathComponent("good.txt").path, &a) == 0 && lstat(out.url.appendingPathComponent("hl-good").path, &b) == 0 && a.st_ino == b.st_ino)166        // 7: the file is still a file with its content; the link "s" was not overwritten nor followed.167        #expect(items["victim.txt"]?.kind == "file" && items["s"]?.kind == "link")168        #expect(try String(contentsOf: out.url.appendingPathComponent("victim.txt"), encoding: .utf8) == "victim")169        #expect(items["sneaky"] == nil)170        // Every remaining link resolves inside the destination.171        let rootPath = try #require(realpath(out.url.path, nil).map { p -> String in defer { free(p) }; return String(cString: p) })172        for (name, item) in items where item.kind == "link" {173            guard let real = realpath(rootPath + "/" + name, nil) else { continue }          // dangling: resolves to nothing at all174            let resolved = String(cString: real)175            free(real)176            #expect(resolved == rootPath || resolved.hasPrefix(rootPath + "/"), "\(name) → \(resolved)")177        }178        #expect(out.collector.message("symlinks-refused").hasPrefix("6 symbolic links point outside"), "\(out.collector.message("symlinks-refused"))")179        #expect(out.collector.message("hardlinks-refused").hasPrefix("3 hard links refer"), "\(out.collector.message("hardlinks-refused"))")180        #expect(out.collector.warnings.filter { $0.code == "entries-rejected" }.count >= 2)181        // With links turned off none is created, and that is reported.182        let none = try await runEngine([archive], .tar, .folder, operation: Self.extract, options: ["symlinks": .string("skip")], name: "none", in: scratch)183        #expect(try Tree.snapshot(none.url).values.allSatisfy { $0.kind != "link" } && none.collector.has("symlinks-skipped"))184        #expect(try Tree.snapshot(outsideDir) == before)185    }186187    @Test func devicesFifosAndSetIDBitsAreNeverCreated() async throws {188        let scratch = try Scratch(); defer { scratch.cleanUp() }189        let E = Craft.TarEntry.self190        let archive = try scratch.file("special.tar", Craft.tar([191            E.init("dev/null-clone", type: "3", mode: 0o666), E.init("dev/disk", type: "4", mode: 0o660), E.init("pipe", type: "6", mode: 0o644),192            E.init("suid-root", mode: 0o4755, data: "#!/bin/sh\n"), E.init("sgid", mode: 0o2755, data: "x"), E.init("sticky-dir/", type: "5", mode: 0o1777),193            E.init("unreadable", mode: 0o000, data: "x"), E.init("plain.txt", mode: 0o640, data: "x"),194        ]))195        let out = try await runEngine([archive], .tar, .folder, operation: Self.extract, name: "out", in: scratch)196        expectAllPass(out.checks, "special")197        let items = try Tree.snapshot(out.url)198        #expect(Set(items.keys) == ["suid-root", "sgid", "sticky-dir", "unreadable", "plain.txt"], "\(items.keys.sorted())")199        #expect(items["suid-root"]?.mode == 0o755 && items["sgid"]?.mode == 0o755 && items["sticky-dir"]?.mode == 0o777 && items["plain.txt"]?.mode == 0o640)200        #expect(items["unreadable"]?.mode == 0o600)          // the owner can always read and delete what was extracted201        #expect(items.values.allSatisfy { $0.kind == "file" || $0.kind == "dir" })202        #expect(out.collector.message("special-files-skipped").hasPrefix("3 device, FIFO or socket entries"))203        #expect(out.collector.message("special-bits-stripped").contains("3 entries"))204        // Ownership is never changed: everything belongs to the user running the tests.205        var info = stat()206        #expect(lstat(out.url.appendingPathComponent("suid-root").path, &info) == 0 && info.st_uid == getuid())207    }208209    /// A tiny archive that expands to `megabytes` of zeros.210    static func bomb(_ format: FormatID, megabytes: Int, in scratch: Scratch) async throws -> URL {211        let folder = try scratch.directory("bomb-src-\(UUID().uuidString.prefix(6))/bomb")212        let zeros = folder.appendingPathComponent("zeros.bin")213        FileManager.default.createFile(atPath: zeros.path, contents: nil)214        let handle = try FileHandle(forWritingTo: zeros)215        try handle.truncate(atOffset: UInt64(megabytes) << 20)          // sparse: no disk used216        try handle.close()217        return try await runEngine([folder], .folder, format, operation: ArchiveEngine.createOperation, options: ["compression.level": .string("fast")], in: scratch).url218    }219220    @Test(arguments: [FormatID.tzst, .sevenZip, .txz])221    func decompressionBombsStopEarlyAndLeaveNothingBehind(format: FormatID) async throws {222        let scratch = try Scratch(); defer { scratch.cleanUp() }223        let archive = try await Self.bomb(format, megabytes: 400, in: scratch)224        let size = try #require(try archive.resourceValues(forKeys: [.fileSizeKey]).fileSize)225        #expect(size < 200_000, "the bomb itself is \(size) bytes")226        func attempt(_ operation: String?, to: FormatID, options: [String: OptionValue] = [:]) async throws -> String {227            let work = try scratch.work()228            defer { #expect((try? FileManager.default.contentsOfDirectory(atPath: work.path))?.isEmpty == true, "partial output left by \(operation ?? "convert")") }229            do {230                _ = try await ArchiveEngine().convert(.init(source: archive, sourceFormat: format, destinationFormat: to, operation: operation, options: ConversionOptions(options), outputBaseName: "out"),231                                                      context: .init(workDirectory: work))232                return "no error"233            } catch ConversionError.limitExceeded(let message) { return message }234        }235        let started = Date()236        // 7z announces its sizes, so it is refused before a byte is written; the tar streams are stopped on the way.237        let extract = try await attempt(Self.extract, to: .folder)238        #expect(extract.contains("decompression bomb") || extract.contains("times its size") || extract.contains("above the limit"), "\(extract)")239        #expect(try await attempt(nil, to: .zip).contains("decompression bomb"))240        #expect(try await attempt(ArchiveEngine.verifyOperation, to: .txt).contains("decompression bomb"))241        #expect(Date().timeIntervalSince(started) < 20)242        #expect(try await attempt(Self.extract, to: .folder, options: ["limit.ratio": .int(0), "limit.total.mb": .int(100)]).contains("100 MB"))243        #expect(try await attempt(Self.extract, to: .folder, options: ["limit.ratio": .int(0), "limit.entry.mb": .int(50)]).contains("50 MB per file"))244    }245246    @Test func trustedHighRatioArchiveExtractsWhenTheLimitIsRaised() async throws {247        let scratch = try Scratch(); defer { scratch.cleanUp() }248        // 7z announces its sizes, so the ratio is known before extracting and preflight can warn about it.249        let sevenZip = try await Self.bomb(.sevenZip, megabytes: 64, in: scratch)250        let inspection = try await ArchiveEngine().inspect(sevenZip, format: .sevenZip)251        #expect(inspection.uncompressedSize == 64 << 20)252        let request = ConversionRequest(source: sevenZip, sourceFormat: .sevenZip, destinationFormat: .folder, operation: Self.extract)253        #expect(ArchiveEngine().preflight(request, inspection: inspection).contains { $0.code == "limit-ratio" })254        let archive = try await Self.bomb(.tzst, megabytes: 64, in: scratch)255        await #expect(throws: ConversionError.self) { _ = try await runEngine([archive], .tzst, .folder, operation: Self.extract, name: "no", in: scratch) }256        let out = try await runEngine([archive], .tzst, .folder, operation: Self.extract, options: ["limit.ratio": .int(0)], name: "yes", in: scratch)257        expectAllPass(out.checks, "raised limit")258        #expect(try out.url.appendingPathComponent("bomb/zeros.bin").resourceValues(forKeys: [.fileSizeKey]).fileSize == 64 << 20)259    }260261    @Test func entryCountAndSizeLimitsApply() async throws {262        let scratch = try Scratch(); defer { scratch.cleanUp() }263        let tree = try Tree.make(at: scratch.url("tree"), bigMegabytes: 3)264        let archive = try #require(try Fixture.system(.tgz, of: tree, in: scratch))265        for (options, needle) in [(["limit.entries": OptionValue.int(5)], "more than 5 entries"), (["limit.total.mb": .int(1)], "more than 1 MB"), (["limit.entry.mb": .int(2)], "2 MB per file")] {266            for (operation, to) in [(Self.extract as String?, FormatID.folder), (nil, .zip), (ArchiveEngine.verifyOperation, .json)] {267                if operation == ArchiveEngine.verifyOperation, needle.contains("per file") { continue }      // verification writes nothing: no per-file limit268                let work = try scratch.work()269                do {270                    _ = try await ArchiveEngine().convert(.init(source: archive, sourceFormat: .tgz, destinationFormat: to, operation: operation, options: ConversionOptions(options), outputBaseName: "o"),271                                                          context: .init(workDirectory: work))272                    Issue.record("\(needle) not enforced for \(operation ?? "convert")")273                } catch ConversionError.limitExceeded(let message) { #expect(message.contains(needle), "\(message)") }274                #expect(try FileManager.default.contentsOfDirectory(atPath: work.path).isEmpty)275            }276        }277        // An archive inside an archive is just a file.278        let nested = try scratch.directory("nest/outer")279        try FileManager.default.copyItem(at: archive, to: nested.appendingPathComponent("inner.tar.gz"))280        let outer = try await runEngine([nested], .folder, .zip, operation: ArchiveEngine.createOperation, in: scratch)281        let out = try await runEngine([outer.url], .zip, .folder, operation: Self.extract, name: "out", in: scratch)282        #expect(try FileManager.default.contentsOfDirectory(atPath: out.url.appendingPathComponent("outer").path) == ["inner.tar.gz"])283        // Disk space: asking for more than the volume has is refused; asking for nothing passes.284        let extractor = try SecureExtractor(root: scratch.url("space"), archive: archive, archiveSize: 1, settings: ArchiveSettings(), flag: CancellationFlag())285        #expect(throws: ConversionError.self) { try extractor.checkDiskSpace(needed: Int64.max / 4) }286        try extractor.checkDiskSpace(needed: 0)287    }288289    @Test(arguments: ArchiveEngine.archiveFormats.filter { $0 != .rar })290    func truncatedAndCorruptedArchivesFailCleanly(format: FormatID) async throws {291        let scratch = try Scratch(); defer { scratch.cleanUp() }292        let good = try await MatrixFixtures.shared.archive(format)293        let bytes = [UInt8](try Data(contentsOf: good))294        var variants: [(String, [UInt8])] = [("cut in half", Array(bytes.prefix(bytes.count / 2))), ("cut at 90 %", Array(bytes.prefix(bytes.count * 9 / 10))), ("empty", []), ("16 bytes", Array(bytes.prefix(16)))]295        var damaged = bytes296        for i in stride(from: bytes.count / 3, to: bytes.count / 3 + 64, by: 1) where i < damaged.count { damaged[i] ^= 0xA5 }297        variants.append(("64 bytes overwritten", damaged))298        variants.append(("random bytes", [UInt8](Tree.random(4096, seed: 99))))299        for (what, content) in variants {300            let broken = try scratch.file("broken-\(UUID().uuidString.prefix(6)).\(Fixture.ext(format))", Data(content))301            let work = try scratch.work()302            var failed = false303            do {304                let output = try await ArchiveEngine().convert(.init(source: broken, sourceFormat: format, destinationFormat: .folder, operation: Self.extract, outputBaseName: "out"),305                                                               context: .init(workDirectory: work))306                // A plain tar cut exactly between entries, or damage inside file data that tar cannot detect, may307                // legitimately succeed; anything with checksums must not.308                #expect(format == .tar, "\(format) \(what): extraction succeeded with \(output.facts)")309            } catch is ConversionError { failed = true }310            if failed { #expect(try FileManager.default.contentsOfDirectory(atPath: work.path).isEmpty, "\(format) \(what): partial output left") }311            // Listing, inspection and verification never crash either.312            _ = try? await ArchiveBrowser.list(broken, format: format)313            _ = try? await ArchiveEngine().inspect(broken, format: format)314            if let report = try? await ArchiveVerifier.verify(broken, format: format), format != .tar { #expect(!report.passed, "\(format) \(what): verification passed") }315        }316    }317318    @Test(arguments: [FormatID.zip, .sevenZip, .tar, .tgz, .rar])319    func mutatedArchivesNeverCrashOrEscape(format: FormatID) async throws {320        let scratch = try Scratch(); defer { scratch.cleanUp() }321        let E = Craft.TarEntry.self322        let seed: [UInt8]323        switch format {324        case .tar: seed = [UInt8](Craft.tar([E.init("a/b.txt", data: "hello"), E.init("l", type: "2", link: "a/b.txt"), E.init("h", type: "1", link: "a/b.txt"), E.init("d/", type: "5", mode: 0o755)]))325        case .zip: seed = [UInt8](Craft.zip([Craft.ZipEntry("a/b.txt", "hello"), Craft.ZipEntry("l", "a/b.txt", unixMode: 0o120777), Craft.ZipEntry("d/", "", unixMode: 0o040755)], comment: "c"))326        case .rar: seed = [UInt8](Craft.rar5(MatrixFixtures.rarFiles))327        default:328            let url = scratch.url("seed.\(Fixture.ext(format))")329            try Self.hostile(format, names: ["a/b.txt", "c.txt", "d/e/f.txt"], to: url)330            seed = [UInt8](try Data(contentsOf: url))331        }332        var rng: UInt64 = UInt64(seed.count) &* 2654435761333        func next(_ bound: Int) -> Int { rng = rng &* 6364136223846793005 &+ 1442695040888963407; return Int((rng >> 33) % UInt64(max(1, bound))) }334        let before = try Self.outside(scratch).filter { !$0.hasPrefix("m-") && !$0.hasPrefix("seed.") }335        var extracted = 0336        for round in 0..<150 {337            var bytes = seed338            switch round % 3 {339            case 0: for _ in 0..<(1 + next(3)) { bytes[next(bytes.count)] = UInt8(next(256)) }340            case 1: for _ in 0..<(1 + next(3)) { let i = next(bytes.count); bytes[i] = [0x2E, 0x2F, 0x5C, 0x00, 0xFF][next(5)] }          // dots, slashes, NULs into names and fields341            default: bytes.removeLast(next(bytes.count))342            }343            let archive = try scratch.file("m-\(round).\(Fixture.ext(format))", Data(bytes))344            let work = try scratch.work()345            if (try? await ArchiveEngine().convert(.init(source: archive, sourceFormat: format, destinationFormat: .folder, operation: Self.extract, outputBaseName: "out"), context: .init(workDirectory: work))) != nil {346                extracted += 1347                #expect(try FileManager.default.contentsOfDirectory(atPath: work.path) == ["out"])348                let failures = OutputValidator.validate(ConversionOutput(file: work.appendingPathComponent("out")), destinationFormat: .folder, workDirectory: work).filter { !$0.passed }349                let links = ((try? Tree.snapshot(work.appendingPathComponent("out"))) ?? [:]).filter { $0.value.kind == "link" }.map { "\($0.key) → \(Array($0.value.target.utf8))" }350                #expect(failures.isEmpty, "\(failures.map(\.detail)) links: \(links)")351            } else {352                #expect(try FileManager.default.contentsOfDirectory(atPath: work.path).isEmpty)353            }354            try? FileManager.default.removeItem(at: archive)355        }356        #expect(try Self.outside(scratch).filter { !$0.hasPrefix("m-") && !$0.hasPrefix("seed.") } == before)357        #expect(extracted > 0, "no mutation survived: the loop did not exercise the extractor")358    }359360    @Test func quarantineFlagFollowsTheArchive() async throws {361        let scratch = try Scratch(); defer { scratch.cleanUp() }362        let tree = try Tree.make(at: scratch.url("tree"), bigMegabytes: 1)363        let archive = try #require(try Fixture.system(.zip, of: tree, in: scratch))364        let flagValue = Data("0083;65a1b2c3;Safari;F643CD5F-6071-46AB-83AB-390BA944DEC5".utf8)365        let plain = try await runEngine([archive], .zip, .folder, operation: Self.extract, name: "plain", in: scratch)366        #expect(SecureExtractor.readXattr(plain.url.appendingPathComponent("tree/hello.txt").path, "com.apple.quarantine") == nil && !plain.collector.has("quarantine-propagated"))367        let set = flagValue.withUnsafeBytes { setxattr(archive.path, "com.apple.quarantine", $0.baseAddress, flagValue.count, 0, 0) }368        #expect(set == 0)369        let flagged = try await runEngine([archive], .zip, .folder, operation: Self.extract, name: "flagged", in: scratch)370        expectAllPass(flagged.checks, "quarantine")371        for name in ["tree/hello.txt", "tree/run.sh", "tree/empty.bin", "tree/dir with spaces/nested/deep/file.dat", "tree/empty-dir", "tree"] {372            #expect(SecureExtractor.readXattr(flagged.url.appendingPathComponent(name).path, "com.apple.quarantine") == flagValue, "\(name)")373        }374        #expect(flagged.collector.message("quarantine-propagated").contains("Gatekeeper"))375        let optedOut = try await runEngine([archive], .zip, .folder, operation: Self.extract, options: ["quarantine": .bool(false)], name: "opted", in: scratch)376        #expect(SecureExtractor.readXattr(optedOut.url.appendingPathComponent("tree/run.sh").path, "com.apple.quarantine") == nil)377    }378379    @Test func cancellationIsPromptAndRemovesPartialOutput() async throws {380        let scratch = try Scratch(); defer { scratch.cleanUp() }381        let archive = try await Self.bomb(.tzst, megabytes: 2000, in: scratch)382        for (operation, to) in [(Self.extract as String?, FormatID.folder), (nil, .tar), (ArchiveEngine.verifyOperation, .txt)] {383            let work = try scratch.work()384            let seen = Collector()385            let started = Date()386            let task = Task {387                try await ArchiveEngine().convert(.init(source: archive, sourceFormat: .tzst, destinationFormat: to, operation: operation,388                                                        options: ConversionOptions(["limit.ratio": .int(0)]), outputBaseName: "out"),389                                                  context: .init(workDirectory: work, progress: { seen.add($0); withUnsafeCurrentTask { _ in } }))390            }391            // Let it write some data, then cancel.392            while seen.progress.isEmpty, Date().timeIntervalSince(started) < 10 { try await Task.sleep(for: .milliseconds(5)) }393            let cancelledAt = Date()394            task.cancel()395            await #expect(throws: CancellationError.self) { _ = try await task.value }396            #expect(Date().timeIntervalSince(cancelledAt) < 2, "\(operation ?? "convert") took \(Date().timeIntervalSince(cancelledAt)) s to stop")397            #expect(try FileManager.default.contentsOfDirectory(atPath: work.path).isEmpty, "\(operation ?? "convert") left partial output")398        }399        // Creation: cancelled while reading a large file.400        let folder = try scratch.directory("big/src")401        FileManager.default.createFile(atPath: folder.appendingPathComponent("zeros").path, contents: nil)402        let handle = try FileHandle(forWritingTo: folder.appendingPathComponent("zeros")); try handle.truncate(atOffset: 3000 << 20); try handle.close()403        let work = try scratch.work()404        let seen = Collector()405        let task = Task { try await ArchiveEngine().convert(.init(source: folder, sourceFormat: .folder, destinationFormat: .zip, operation: ArchiveEngine.createOperation),406                                                            context: .init(workDirectory: work, progress: { seen.add($0) })) }407        let started = Date()408        while seen.progress.isEmpty, Date().timeIntervalSince(started) < 10 { try await Task.sleep(for: .milliseconds(5)) }409        task.cancel()410        await #expect(throws: CancellationError.self) { _ = try await task.value }411        #expect(try FileManager.default.contentsOfDirectory(atPath: work.path).isEmpty)412    }413}414
Files main
Menu

Theme