Repository Public License
1// ORIX Convert — Packages/EngineArchive/Tests/EngineArchiveTests/ListingTests.swift2// Description: The archive browser (paged listing, summary), inspection, integrity verification that really detects3// damage, the RAR fixtures assembled from the format specification, and the name sanitiser.4// Author: Simon-Pierre Boucher — contact@spboucher.ai5import ConvertCore6@testable import EngineArchive7import Foundation8import Testing910@Suite struct ListingTests {11 @Test func browserPagesThroughALargeListing() async throws {12 let scratch = try Scratch(); defer { scratch.cleanUp() }13 let folder = try scratch.directory("many/files")14 for i in 0..<250 { try Data("file \(i)\n".utf8).write(to: folder.appendingPathComponent(String(format: "f%03d.txt", i))) }15 for format in [FormatID.zip, .sevenZip, .txz] {16 let archive = try await runEngine([folder], .folder, format, operation: ArchiveEngine.createOperation, in: scratch).url17 let page = try await ArchiveBrowser.list(archive, format: format, offset: 101, limit: 50)18 #expect(page.entries.count == 50 && page.entries.first?.index == 101 && page.entries.last?.index == 150, "\(format)")19 // 7z lists folders after files, so the same index is one file further.20 #expect(page.entries.first?.path == (format == .sevenZip ? "files/f101.txt" : "files/f100.txt") && page.entries.first?.kind == .file && page.entries.first?.size == 9, "\(format): \(String(describing: page.entries.first))")21 #expect(page.summary.entryCount == 251 && page.summary.fileCount == 250 && page.summary.directoryCount == 1 && page.summary.isComplete, "\(format)")22 #expect(page.summary.totalSize == (0..<250).reduce(Int64(0)) { $0 + Int64("file \($1)\n".utf8.count) } && page.summary.sizesKnown)23 #expect(try await ArchiveBrowser.list(archive, format: format, offset: 240, limit: 50).entries.count == 11)24 #expect(try await ArchiveBrowser.list(archive, format: format, offset: 9999, limit: 50).entries.isEmpty)25 let bounded = try await ArchiveBrowser.list(archive, format: format, offset: 0, limit: 10, scanLimit: 40)26 #expect(!bounded.summary.isComplete && bounded.summary.entryCount == 40 && bounded.summary.compressionRatio == nil)27 if format == .zip { #expect(page.entries.allSatisfy { $0.compressedSize != nil }) } else { #expect(page.entries.allSatisfy { $0.compressedSize == nil }) }28 #expect(page.entries.allSatisfy { $0.modified != nil && $0.hasSafeName && !$0.isEncrypted })29 }30 }3132 @Test func summaryReadsCommentSizesAndRatio() async throws {33 let scratch = try Scratch(); defer { scratch.cleanUp() }34 let crafted = try scratch.file("c.zip", Craft.zip([Craft.ZipEntry("a.txt", "aaaa"), Craft.ZipEntry("dir/", "", unixMode: 0o040755), Craft.ZipEntry("dir/l", "../a.txt", unixMode: 0o120777)],35 comment: "Made for the tests — ZIP comment"))36 let page = try await ArchiveBrowser.list(crafted, format: .zip)37 #expect(page.summary.comment == "Made for the tests — ZIP comment" && page.summary.format.hasPrefix("ZIP") && page.summary.filters.isEmpty)38 #expect(page.entries.map(\.kind) == [.file, .directory, .symlink] && page.entries[2].linkTarget == "../a.txt" && page.entries[0].compressedSize == 4)39 #expect(page.entries[0].modified == Date(timeIntervalSince1970: TimeInterval(page.entries[0].modified?.timeIntervalSince1970 ?? 0)) && page.entries[0].permissions == 0o644)40 let tree = try Tree.make(at: scratch.url("tree"), bigMegabytes: 4)41 let system = try #require(try Fixture.system(.zip, of: tree, in: scratch))42 let engine = ArchiveEngine()43 let i = try await engine.inspect(system, format: .zip)44 let fields = Dictionary(i.fields.map { ($0.key, $0.value) }, uniquingKeysWith: { a, _ in a })45 #expect(i.entryCount == 15 && fields["Files"] == "8" && fields["Folders"] == "5" && fields["Links"] == "2" && i.isEncrypted == false)46 let expectedSize = try Tree.snapshot(tree).values.reduce(Int64(0)) { $0 + $1.size }47 #expect(i.uncompressedSize == expectedSize)48 #expect(fields["Compression ratio"] != nil && fields["First entries"]?.contains("tree/hello.txt") == true && fields["Encrypted entries"] == "none seen")49 let ratio = try #require(try await ArchiveBrowser.list(system, format: .zip).summary.compressionRatio)50 #expect(ratio > 5)51 // A compressed tar names its filter; a single stream cannot announce its size.52 let tgz = try #require(try Fixture.system(.tgz, of: tree, in: scratch))53 let t = try await engine.inspect(tgz, format: .tgz)54 #expect(t.fields.contains { $0.key == "Compression filter" && $0.value == "gzip" } && t.entryCount == 15)55 let gz = try #require(try Fixture.systemStream(.gz, of: tree.appendingPathComponent("big.log"), in: scratch))56 let g = try await engine.inspect(gz, format: .gz)57 #expect(g.entryCount == 1 && g.uncompressedSize == nil && g.notes.contains { $0.contains("not stored in this file") })58 #expect(try await ArchiveBrowser.list(gz, format: .gz).entries.map(\.path) == ["big.log"])59 // Not an archive at all.60 let text = try scratch.file("plain.gz", Data("just text".utf8))61 await #expect(throws: ConversionError.self) { _ = try await engine.inspect(text, format: .gz) }62 await #expect(throws: ConversionError.self) { _ = try await engine.inspect(text, format: .zip) }63 await #expect(throws: ConversionError.self) { _ = try await engine.inspect(scratch.url("missing.zip"), format: .zip) }64 }6566 @Test func verificationFindsDamageInsideAnArchive() async throws {67 let scratch = try Scratch(); defer { scratch.cleanUp() }68 // A stored ZIP: flipping one byte of a file's data changes nothing structural, only the content.69 let payload = String(repeating: "integrity matters. ", count: 400)70 var bytes = [UInt8](Craft.zip([Craft.ZipEntry("first.txt", "untouched"), Craft.ZipEntry("second.txt", payload), Craft.ZipEntry("third.txt", "also untouched")]))71 let at = try #require(Data(bytes).range(of: Data("integrity matters. integrity".utf8))?.lowerBound)72 bytes[at + 200] ^= 0x0173 let damaged = try scratch.file("damaged.zip", Data(bytes))74 let report = try await ArchiveVerifier.verify(damaged, format: .zip)75 #expect(!report.passed && report.failedEntries == 1 && report.checkedEntries == 3)76 #expect(report.entries.first { !$0.ok }?.path == "second.txt" && report.entries.filter(\.ok).map(\.path) == ["first.txt", "third.txt"])77 // Through the engine: the job delivers a report and warns; it does not pretend the archive is fine.78 for destination in [FormatID.txt, .json] {79 let out = try await runEngine([damaged], .zip, destination, operation: ArchiveEngine.verifyOperation, in: scratch)80 expectAllPass(out.checks, "report \(destination)")81 #expect(out.output.facts["passed"] == "no" && out.output.facts["failed"] == "1" && out.collector.message("verification-failed").hasPrefix("1 of 3 files failed"))82 let text = try String(contentsOf: out.url, encoding: .utf8)83 #expect(destination == .txt ? text.contains("Result: FAILED") && text.contains("FAILED") && text.contains("second.txt") : text.contains("\"passed\" : false"))84 }85 // The same damage is refused by extraction and by conversion.86 await #expect(throws: ConversionError.self) { _ = try await runEngine([damaged], .zip, .folder, operation: ArchiveEngine.extractOperation, name: "o", in: scratch) }87 await #expect(throws: ConversionError.self) { _ = try await runEngine([damaged], .zip, .tar, operation: nil, in: scratch) }88 // gzip: the trailer's CRC is checked by this engine (the decoder library does not).89 let log = try scratch.file("data.log", Tree.bigLog(megabytes: 1))90 let gz = try #require(try Fixture.systemStream(.gz, of: log, in: scratch))91 var gzBytes = [UInt8](try Data(contentsOf: gz))92 #expect(try await ArchiveVerifier.verify(gz, format: .gz).passed)93 gzBytes[gzBytes.count - 6] ^= 0xFF // the stored CRC-32 itself94 let badCRC = try scratch.file("crc.gz", Data(gzBytes))95 let gzReport = try await ArchiveVerifier.verify(badCRC, format: .gz)96 #expect(!gzReport.passed && gzReport.fatalError?.contains("incorrect data check") == true)97 await #expect(throws: ConversionError.self) { _ = try await runEngine([badCRC], .gz, .folder, operation: ArchiveEngine.extractOperation, name: "g", in: scratch) }98 await #expect(throws: ConversionError.self) { _ = try await runEngine([badCRC], .gz, .xz, operation: nil, in: scratch) }99 // Two gzip members one after the other are one valid file.100 let twice = try scratch.file("twice.gz", try Data(contentsOf: gz) + Data(contentsOf: gz))101 #expect(try await ArchiveVerifier.verify(twice, format: .gz).passed)102 // TAR stores no checksum of file data, and the report says so instead of claiming a pass means "intact".103 let tar = try scratch.file("plain.tar", Craft.tar([Craft.TarEntry("a.txt", data: "abc")]))104 let tarReport = try await ArchiveVerifier.verify(tar, format: .tar)105 #expect(tarReport.passed && tarReport.coverage.contains("no checksum of file data"))106 }107108 @Test(arguments: [4, 5])109 func rarArchivesBuiltFromTheSpecificationAreRead(version: Int) async throws {110 let scratch = try Scratch(); defer { scratch.cleanUp() }111 let files = MatrixFixtures.rarFiles112 let data = version == 4 ? Craft.rar4(files) : Craft.rar5(files)113 let rar = try scratch.file("v\(version).rar", data)114 #expect(FileTypeDetector.detect(rar).format == .rar)115 let page = try await ArchiveBrowser.list(rar, format: .rar)116 #expect(page.summary.format.hasPrefix(version == 4 ? "RAR" : "RAR5") && page.entries.map(\.path) == files.map(\.name))117 #expect(page.entries.map(\.size) == files.map { Int64($0.data.utf8.count) } && !page.summary.hasEncryptedEntries)118 #expect(page.entries[0].modified == Date(timeIntervalSince1970: TimeInterval(Tree.baseTime)) || version == 4) // RAR 4 stores local DOS time119 let report = try await ArchiveVerifier.verify(rar, format: .rar)120 #expect(report.passed && report.checkedEntries == 3 && report.entries[0].crc32 == String(format: "%08x", Craft.crc(Array(files[0].data.utf8))))121 // A wrong byte in a stored file is caught by the RAR CRC.122 var bytes = [UInt8](data)123 let at = try #require(data.range(of: Data("Stored in a RAR".utf8))?.lowerBound)124 bytes[at] ^= 0x20125 let damaged = try scratch.file("damaged.rar", Data(bytes))126 let bad = try await ArchiveVerifier.verify(damaged, format: .rar)127 #expect(!bad.passed && bad.failedEntries == 1 && bad.entries[0].message?.contains("RAR header says") == true)128 await #expect(throws: ConversionError.self) { _ = try await runEngine([damaged], .rar, .zip, operation: nil, in: scratch) }129 let directory = try #require(RarDirectory.read(rar))130 #expect(directory.records.map(\.size) == files.map { Int64($0.data.utf8.count) } && directory.records.map(\.crc) == files.map { Craft.crc(Array($0.data.utf8)) })131 await #expect(throws: ConversionError.self) { _ = try await runEngine([damaged], .rar, .folder, operation: ArchiveEngine.extractOperation, name: "o", in: scratch) }132 // A header CRC that does not match is refused as well.133 var header = [UInt8](data)134 header[version == 4 ? 9 : 12] ^= 0xFF135 let broken = try scratch.file("header.rar", Data(header))136 await #expect(throws: ConversionError.self) { _ = try await runEngine([broken], .rar, .folder, operation: ArchiveEngine.extractOperation, name: "h", in: scratch) }137 }138139 @Test func sanitiserRules() {140 func ok(_ raw: String) -> String? { if case .success(let r) = PathSanitizer.sanitize(raw) { return r.path }; return nil }141 func modified(_ raw: String) -> Bool? { if case .success(let r) = PathSanitizer.sanitize(raw) { return r.wasModified }; return nil }142 #expect(ok("a/b/c.txt") == "a/b/c.txt" && modified("a/b/c.txt") == false && modified("dir/") == false && ok("dir/") == "dir")143 #expect(ok("/etc/passwd") == "etc/passwd" && modified("/etc/passwd") == true && ok("//server/share/x") == "server/share/x")144 #expect(ok("C:\\Users\\x.txt") == "Users/x.txt" && ok("c:x.txt") == "x.txt" && ok("\\\\host\\share\\f") == "host/share/f")145 #expect(ok("./a/./b") == "a/b" && ok("a//b") == "a/b")146 for bad in ["..", "../x", "a/../b", "a/..", "..\\x", "a\\..\\..\\b", "", "/", "./.", "a/..namedfork/rsrc", "x/..NamedFork/data"] { #expect(ok(bad) == nil, "\(bad)") }147 #expect(ok("..x/y..") == "..x/y.." && ok("...") == "_.." && ok(". .") == "_ .")148 #expect(ok("a\u{0}b") == "a_b" && ok("tab\there") == "tab_here" && ok("rtl\u{202E}txt.exe") == "rtl_txt.exe" && ok("bom\u{FEFF}") == "bom_")149 #expect(ok(String(repeating: "é", count: 127)) != nil && ok(String(repeating: "é", count: 128)) == nil) // 255 bytes, not characters150 #expect(ok((0..<128).map { _ in "d" }.joined(separator: "/")) != nil && ok((0..<129).map { _ in "d" }.joined(separator: "/")) == nil)151 #expect(ok(String(repeating: "abcdefghij/", count: 400)) == nil)152 // Same file on a case- and normalisation-insensitive volume.153 #expect(PathSanitizer.collisionKey(["Caf\u{00E9}", "A.TXT"]) == PathSanitizer.collisionKey(["cafe\u{0301}", "a.txt"]))154 #expect(PathSanitizer.linkStaysInside(linkComponents: ["a", "b", "link"], target: "../c") && PathSanitizer.linkStaysInside(linkComponents: ["a", "link"], target: "..")155 && !PathSanitizer.linkStaysInside(linkComponents: ["a", "link"], target: "../..") && !PathSanitizer.linkStaysInside(linkComponents: ["link"], target: "/etc")156 && !PathSanitizer.linkStaysInside(linkComponents: ["link"], target: "") && !PathSanitizer.linkStaysInside(linkComponents: ["a", "link"], target: "x/../../../y"))157 #expect(PathSanitizer.isMacOSNoise(["a", ".DS_Store"]) && PathSanitizer.isMacOSNoise(["__MACOSX", "a", "._b"]) && PathSanitizer.isMacOSNoise(["a", "._b"])158 && !PathSanitizer.isMacOSNoise(["a", "b.DS_Store"]) && !PathSanitizer.isMacOSNoise(["_MACOSX", "x"]))159 }160161 @Test func danglingLinksAreNotKeptAndThatIsReported() async throws {162 let scratch = try Scratch(); defer { scratch.cleanUp() }163 let E = Craft.TarEntry.self164 let archive = try scratch.file("d.tar", Craft.tar([E.init("real.txt", data: "x"), E.init("ok", type: "2", link: "real.txt"), E.init("broken", type: "2", link: "missing/file.txt"),165 E.init("later", type: "2", link: "sub/late.txt"), E.init("sub/late.txt", data: "appears after its link")]))166 let out = try await runEngine([archive], .tar, .folder, operation: ArchiveEngine.extractOperation, name: "out", in: scratch)167 expectAllPass(out.checks, "dangling")168 let items = try Tree.snapshot(out.url)169 #expect(items["ok"]?.kind == "link" && items["later"]?.kind == "link" && items["broken"] == nil)170 #expect(out.collector.message("symlinks-dangling").hasPrefix("1 symbolic link points to something that is not in the archive"))171 #expect(out.output.facts["links"] == "2")172 }173}174