SPB Git

Repository Public License

ListingTests.swift 15.3 KB 174 lines swift

1// ORIX Convert — Packages/EngineArchive/Tests/EngineArchiveTests/ListingTests.swift2// Description: The archive browser (paged listing, summary), inspection, integrity verification that really detects3//              damage, the RAR fixtures assembled from the format specification, and the name sanitiser.4// Author: Simon-Pierre Boucher — contact@spboucher.ai5import ConvertCore6@testable import EngineArchive7import Foundation8import Testing910@Suite struct ListingTests {11    @Test func browserPagesThroughALargeListing() async throws {12        let scratch = try Scratch(); defer { scratch.cleanUp() }13        let folder = try scratch.directory("many/files")14        for i in 0..<250 { try Data("file \(i)\n".utf8).write(to: folder.appendingPathComponent(String(format: "f%03d.txt", i))) }15        for format in [FormatID.zip, .sevenZip, .txz] {16            let archive = try await runEngine([folder], .folder, format, operation: ArchiveEngine.createOperation, in: scratch).url17            let page = try await ArchiveBrowser.list(archive, format: format, offset: 101, limit: 50)18            #expect(page.entries.count == 50 && page.entries.first?.index == 101 && page.entries.last?.index == 150, "\(format)")19            // 7z lists folders after files, so the same index is one file further.20            #expect(page.entries.first?.path == (format == .sevenZip ? "files/f101.txt" : "files/f100.txt") && page.entries.first?.kind == .file && page.entries.first?.size == 9, "\(format): \(String(describing: page.entries.first))")21            #expect(page.summary.entryCount == 251 && page.summary.fileCount == 250 && page.summary.directoryCount == 1 && page.summary.isComplete, "\(format)")22            #expect(page.summary.totalSize == (0..<250).reduce(Int64(0)) { $0 + Int64("file \($1)\n".utf8.count) } && page.summary.sizesKnown)23            #expect(try await ArchiveBrowser.list(archive, format: format, offset: 240, limit: 50).entries.count == 11)24            #expect(try await ArchiveBrowser.list(archive, format: format, offset: 9999, limit: 50).entries.isEmpty)25            let bounded = try await ArchiveBrowser.list(archive, format: format, offset: 0, limit: 10, scanLimit: 40)26            #expect(!bounded.summary.isComplete && bounded.summary.entryCount == 40 && bounded.summary.compressionRatio == nil)27            if format == .zip { #expect(page.entries.allSatisfy { $0.compressedSize != nil }) } else { #expect(page.entries.allSatisfy { $0.compressedSize == nil }) }28            #expect(page.entries.allSatisfy { $0.modified != nil && $0.hasSafeName && !$0.isEncrypted })29        }30    }3132    @Test func summaryReadsCommentSizesAndRatio() async throws {33        let scratch = try Scratch(); defer { scratch.cleanUp() }34        let crafted = try scratch.file("c.zip", Craft.zip([Craft.ZipEntry("a.txt", "aaaa"), Craft.ZipEntry("dir/", "", unixMode: 0o040755), Craft.ZipEntry("dir/l", "../a.txt", unixMode: 0o120777)],35                                                          comment: "Made for the tests — ZIP comment"))36        let page = try await ArchiveBrowser.list(crafted, format: .zip)37        #expect(page.summary.comment == "Made for the tests — ZIP comment" && page.summary.format.hasPrefix("ZIP") && page.summary.filters.isEmpty)38        #expect(page.entries.map(\.kind) == [.file, .directory, .symlink] && page.entries[2].linkTarget == "../a.txt" && page.entries[0].compressedSize == 4)39        #expect(page.entries[0].modified == Date(timeIntervalSince1970: TimeInterval(page.entries[0].modified?.timeIntervalSince1970 ?? 0)) && page.entries[0].permissions == 0o644)40        let tree = try Tree.make(at: scratch.url("tree"), bigMegabytes: 4)41        let system = try #require(try Fixture.system(.zip, of: tree, in: scratch))42        let engine = ArchiveEngine()43        let i = try await engine.inspect(system, format: .zip)44        let fields = Dictionary(i.fields.map { ($0.key, $0.value) }, uniquingKeysWith: { a, _ in a })45        #expect(i.entryCount == 15 && fields["Files"] == "8" && fields["Folders"] == "5" && fields["Links"] == "2" && i.isEncrypted == false)46        let expectedSize = try Tree.snapshot(tree).values.reduce(Int64(0)) { $0 + $1.size }47        #expect(i.uncompressedSize == expectedSize)48        #expect(fields["Compression ratio"] != nil && fields["First entries"]?.contains("tree/hello.txt") == true && fields["Encrypted entries"] == "none seen")49        let ratio = try #require(try await ArchiveBrowser.list(system, format: .zip).summary.compressionRatio)50        #expect(ratio > 5)51        // A compressed tar names its filter; a single stream cannot announce its size.52        let tgz = try #require(try Fixture.system(.tgz, of: tree, in: scratch))53        let t = try await engine.inspect(tgz, format: .tgz)54        #expect(t.fields.contains { $0.key == "Compression filter" && $0.value == "gzip" } && t.entryCount == 15)55        let gz = try #require(try Fixture.systemStream(.gz, of: tree.appendingPathComponent("big.log"), in: scratch))56        let g = try await engine.inspect(gz, format: .gz)57        #expect(g.entryCount == 1 && g.uncompressedSize == nil && g.notes.contains { $0.contains("not stored in this file") })58        #expect(try await ArchiveBrowser.list(gz, format: .gz).entries.map(\.path) == ["big.log"])59        // Not an archive at all.60        let text = try scratch.file("plain.gz", Data("just text".utf8))61        await #expect(throws: ConversionError.self) { _ = try await engine.inspect(text, format: .gz) }62        await #expect(throws: ConversionError.self) { _ = try await engine.inspect(text, format: .zip) }63        await #expect(throws: ConversionError.self) { _ = try await engine.inspect(scratch.url("missing.zip"), format: .zip) }64    }6566    @Test func verificationFindsDamageInsideAnArchive() async throws {67        let scratch = try Scratch(); defer { scratch.cleanUp() }68        // A stored ZIP: flipping one byte of a file's data changes nothing structural, only the content.69        let payload = String(repeating: "integrity matters. ", count: 400)70        var bytes = [UInt8](Craft.zip([Craft.ZipEntry("first.txt", "untouched"), Craft.ZipEntry("second.txt", payload), Craft.ZipEntry("third.txt", "also untouched")]))71        let at = try #require(Data(bytes).range(of: Data("integrity matters. integrity".utf8))?.lowerBound)72        bytes[at + 200] ^= 0x0173        let damaged = try scratch.file("damaged.zip", Data(bytes))74        let report = try await ArchiveVerifier.verify(damaged, format: .zip)75        #expect(!report.passed && report.failedEntries == 1 && report.checkedEntries == 3)76        #expect(report.entries.first { !$0.ok }?.path == "second.txt" && report.entries.filter(\.ok).map(\.path) == ["first.txt", "third.txt"])77        // Through the engine: the job delivers a report and warns; it does not pretend the archive is fine.78        for destination in [FormatID.txt, .json] {79            let out = try await runEngine([damaged], .zip, destination, operation: ArchiveEngine.verifyOperation, in: scratch)80            expectAllPass(out.checks, "report \(destination)")81            #expect(out.output.facts["passed"] == "no" && out.output.facts["failed"] == "1" && out.collector.message("verification-failed").hasPrefix("1 of 3 files failed"))82            let text = try String(contentsOf: out.url, encoding: .utf8)83            #expect(destination == .txt ? text.contains("Result: FAILED") && text.contains("FAILED") && text.contains("second.txt") : text.contains("\"passed\" : false"))84        }85        // The same damage is refused by extraction and by conversion.86        await #expect(throws: ConversionError.self) { _ = try await runEngine([damaged], .zip, .folder, operation: ArchiveEngine.extractOperation, name: "o", in: scratch) }87        await #expect(throws: ConversionError.self) { _ = try await runEngine([damaged], .zip, .tar, operation: nil, in: scratch) }88        // gzip: the trailer's CRC is checked by this engine (the decoder library does not).89        let log = try scratch.file("data.log", Tree.bigLog(megabytes: 1))90        let gz = try #require(try Fixture.systemStream(.gz, of: log, in: scratch))91        var gzBytes = [UInt8](try Data(contentsOf: gz))92        #expect(try await ArchiveVerifier.verify(gz, format: .gz).passed)93        gzBytes[gzBytes.count - 6] ^= 0xFF                                    // the stored CRC-32 itself94        let badCRC = try scratch.file("crc.gz", Data(gzBytes))95        let gzReport = try await ArchiveVerifier.verify(badCRC, format: .gz)96        #expect(!gzReport.passed && gzReport.fatalError?.contains("incorrect data check") == true)97        await #expect(throws: ConversionError.self) { _ = try await runEngine([badCRC], .gz, .folder, operation: ArchiveEngine.extractOperation, name: "g", in: scratch) }98        await #expect(throws: ConversionError.self) { _ = try await runEngine([badCRC], .gz, .xz, operation: nil, in: scratch) }99        // Two gzip members one after the other are one valid file.100        let twice = try scratch.file("twice.gz", try Data(contentsOf: gz) + Data(contentsOf: gz))101        #expect(try await ArchiveVerifier.verify(twice, format: .gz).passed)102        // TAR stores no checksum of file data, and the report says so instead of claiming a pass means "intact".103        let tar = try scratch.file("plain.tar", Craft.tar([Craft.TarEntry("a.txt", data: "abc")]))104        let tarReport = try await ArchiveVerifier.verify(tar, format: .tar)105        #expect(tarReport.passed && tarReport.coverage.contains("no checksum of file data"))106    }107108    @Test(arguments: [4, 5])109    func rarArchivesBuiltFromTheSpecificationAreRead(version: Int) async throws {110        let scratch = try Scratch(); defer { scratch.cleanUp() }111        let files = MatrixFixtures.rarFiles112        let data = version == 4 ? Craft.rar4(files) : Craft.rar5(files)113        let rar = try scratch.file("v\(version).rar", data)114        #expect(FileTypeDetector.detect(rar).format == .rar)115        let page = try await ArchiveBrowser.list(rar, format: .rar)116        #expect(page.summary.format.hasPrefix(version == 4 ? "RAR" : "RAR5") && page.entries.map(\.path) == files.map(\.name))117        #expect(page.entries.map(\.size) == files.map { Int64($0.data.utf8.count) } && !page.summary.hasEncryptedEntries)118        #expect(page.entries[0].modified == Date(timeIntervalSince1970: TimeInterval(Tree.baseTime)) || version == 4)       // RAR 4 stores local DOS time119        let report = try await ArchiveVerifier.verify(rar, format: .rar)120        #expect(report.passed && report.checkedEntries == 3 && report.entries[0].crc32 == String(format: "%08x", Craft.crc(Array(files[0].data.utf8))))121        // A wrong byte in a stored file is caught by the RAR CRC.122        var bytes = [UInt8](data)123        let at = try #require(data.range(of: Data("Stored in a RAR".utf8))?.lowerBound)124        bytes[at] ^= 0x20125        let damaged = try scratch.file("damaged.rar", Data(bytes))126        let bad = try await ArchiveVerifier.verify(damaged, format: .rar)127        #expect(!bad.passed && bad.failedEntries == 1 && bad.entries[0].message?.contains("RAR header says") == true)128        await #expect(throws: ConversionError.self) { _ = try await runEngine([damaged], .rar, .zip, operation: nil, in: scratch) }129        let directory = try #require(RarDirectory.read(rar))130        #expect(directory.records.map(\.size) == files.map { Int64($0.data.utf8.count) } && directory.records.map(\.crc) == files.map { Craft.crc(Array($0.data.utf8)) })131        await #expect(throws: ConversionError.self) { _ = try await runEngine([damaged], .rar, .folder, operation: ArchiveEngine.extractOperation, name: "o", in: scratch) }132        // A header CRC that does not match is refused as well.133        var header = [UInt8](data)134        header[version == 4 ? 9 : 12] ^= 0xFF135        let broken = try scratch.file("header.rar", Data(header))136        await #expect(throws: ConversionError.self) { _ = try await runEngine([broken], .rar, .folder, operation: ArchiveEngine.extractOperation, name: "h", in: scratch) }137    }138139    @Test func sanitiserRules() {140        func ok(_ raw: String) -> String? { if case .success(let r) = PathSanitizer.sanitize(raw) { return r.path }; return nil }141        func modified(_ raw: String) -> Bool? { if case .success(let r) = PathSanitizer.sanitize(raw) { return r.wasModified }; return nil }142        #expect(ok("a/b/c.txt") == "a/b/c.txt" && modified("a/b/c.txt") == false && modified("dir/") == false && ok("dir/") == "dir")143        #expect(ok("/etc/passwd") == "etc/passwd" && modified("/etc/passwd") == true && ok("//server/share/x") == "server/share/x")144        #expect(ok("C:\\Users\\x.txt") == "Users/x.txt" && ok("c:x.txt") == "x.txt" && ok("\\\\host\\share\\f") == "host/share/f")145        #expect(ok("./a/./b") == "a/b" && ok("a//b") == "a/b")146        for bad in ["..", "../x", "a/../b", "a/..", "..\\x", "a\\..\\..\\b", "", "/", "./.", "a/..namedfork/rsrc", "x/..NamedFork/data"] { #expect(ok(bad) == nil, "\(bad)") }147        #expect(ok("..x/y..") == "..x/y.." && ok("...") == "_.." && ok(". .") == "_ .")148        #expect(ok("a\u{0}b") == "a_b" && ok("tab\there") == "tab_here" && ok("rtl\u{202E}txt.exe") == "rtl_txt.exe" && ok("bom\u{FEFF}") == "bom_")149        #expect(ok(String(repeating: "é", count: 127)) != nil && ok(String(repeating: "é", count: 128)) == nil)             // 255 bytes, not characters150        #expect(ok((0..<128).map { _ in "d" }.joined(separator: "/")) != nil && ok((0..<129).map { _ in "d" }.joined(separator: "/")) == nil)151        #expect(ok(String(repeating: "abcdefghij/", count: 400)) == nil)152        // Same file on a case- and normalisation-insensitive volume.153        #expect(PathSanitizer.collisionKey(["Caf\u{00E9}", "A.TXT"]) == PathSanitizer.collisionKey(["cafe\u{0301}", "a.txt"]))154        #expect(PathSanitizer.linkStaysInside(linkComponents: ["a", "b", "link"], target: "../c") && PathSanitizer.linkStaysInside(linkComponents: ["a", "link"], target: "..")155                && !PathSanitizer.linkStaysInside(linkComponents: ["a", "link"], target: "../..") && !PathSanitizer.linkStaysInside(linkComponents: ["link"], target: "/etc")156                && !PathSanitizer.linkStaysInside(linkComponents: ["link"], target: "") && !PathSanitizer.linkStaysInside(linkComponents: ["a", "link"], target: "x/../../../y"))157        #expect(PathSanitizer.isMacOSNoise(["a", ".DS_Store"]) && PathSanitizer.isMacOSNoise(["__MACOSX", "a", "._b"]) && PathSanitizer.isMacOSNoise(["a", "._b"])158                && !PathSanitizer.isMacOSNoise(["a", "b.DS_Store"]) && !PathSanitizer.isMacOSNoise(["_MACOSX", "x"]))159    }160161    @Test func danglingLinksAreNotKeptAndThatIsReported() async throws {162        let scratch = try Scratch(); defer { scratch.cleanUp() }163        let E = Craft.TarEntry.self164        let archive = try scratch.file("d.tar", Craft.tar([E.init("real.txt", data: "x"), E.init("ok", type: "2", link: "real.txt"), E.init("broken", type: "2", link: "missing/file.txt"),165                                                         E.init("later", type: "2", link: "sub/late.txt"), E.init("sub/late.txt", data: "appears after its link")]))166        let out = try await runEngine([archive], .tar, .folder, operation: ArchiveEngine.extractOperation, name: "out", in: scratch)167        expectAllPass(out.checks, "dangling")168        let items = try Tree.snapshot(out.url)169        #expect(items["ok"]?.kind == "link" && items["later"]?.kind == "link" && items["broken"] == nil)170        #expect(out.collector.message("symlinks-dangling").hasPrefix("1 symbolic link points to something that is not in the archive"))171        #expect(out.output.facts["links"] == "2")172    }173}174
Files main
Menu

Theme