Repository Public License
1// ORIX Convert — Packages/EngineArchive/Tests/EngineArchiveTests/Support.swift2// Description: Shared test support for the archive engine: a file tree with awkward names and modes, tree comparison,3// system tools used ONLY to make and check fixtures at test time, and byte-exact builders of TAR, ZIP4// and RAR archives (including hostile ones).5// Author: Simon-Pierre Boucher — contact@spboucher.ai6import ConvertCore7@testable import EngineArchive8import Foundation9import Testing10import zlib1112final class Collector: @unchecked Sendable {13 private let lock = NSLock()14 private var items: [ConversionWarning] = []15 private var fractions: [Double] = []16 func add(_ w: ConversionWarning) { lock.lock(); items.append(w); lock.unlock() }17 func add(_ p: ProgressUpdate) { if let f = p.fraction { lock.lock(); fractions.append(f); lock.unlock() } }18 var warnings: [ConversionWarning] { lock.lock(); defer { lock.unlock() }; return items }19 var progress: [Double] { lock.lock(); defer { lock.unlock() }; return fractions }20 func has(_ code: String) -> Bool { warnings.contains { $0.code == code } }21 func message(_ code: String) -> String { warnings.first { $0.code == code }?.message ?? "" }22}2324struct Scratch {25 let root: URL26 init() throws {27 // Under $TMPDIR as it is spelled (/var/folders/…), like the app's own work directory.28 let base = FileManager.default.temporaryDirectory29 root = base.appendingPathComponent("orix-archive-tests-\(UUID().uuidString)", isDirectory: true)30 try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true)31 }32 func url(_ name: String) -> URL { root.appendingPathComponent(name) }33 func file(_ name: String, _ data: Data) throws -> URL {34 let u = url(name)35 try FileManager.default.createDirectory(at: u.deletingLastPathComponent(), withIntermediateDirectories: true)36 try data.write(to: u)37 return u38 }39 func directory(_ name: String) throws -> URL {40 let u = url(name)41 try FileManager.default.createDirectory(at: u, withIntermediateDirectories: true)42 return u43 }44 func work() throws -> URL { try directory("work-\(UUID().uuidString)") }45 func cleanUp() {46 // Folders extracted with read-only modes must still be removable.47 _ = try? Tool.run("/bin/chmod", ["-R", "u+rwx", root.path])48 try? FileManager.default.removeItem(at: root)49 }50}5152/// System tools, used only to create fixtures and to check the engine's output independently.53enum Tool {54 struct Failure: Error, CustomStringConvertible { let description: String }5556 @discardableResult57 static func run(_ tool: String, _ arguments: [String], cwd: URL? = nil, stdin: Data? = nil) throws -> Data {58 let process = Process()59 process.executableURL = URL(fileURLWithPath: tool)60 process.arguments = arguments61 process.currentDirectoryURL = cwd62 process.environment = ["PATH": "/usr/bin:/bin", "LC_ALL": "en_US.UTF-8", "HOME": NSTemporaryDirectory()]63 // stderr goes to a file: reading two pipes would need a second thread, and the test runner's thread pool is64 // already busy with other tests blocked in tools.65 let errorURL = FileManager.default.temporaryDirectory.appendingPathComponent("orix-tool-stderr-\(UUID().uuidString)")66 FileManager.default.createFile(atPath: errorURL.path, contents: nil)67 defer { try? FileManager.default.removeItem(at: errorURL) }68 let out = Pipe()69 process.standardOutput = out70 process.standardError = try FileHandle(forWritingTo: errorURL)71 let input = stdin == nil ? nil : Pipe()72 process.standardInput = input ?? FileHandle.nullDevice73 try process.run()74 if let stdin, let input { input.fileHandleForWriting.write(stdin); try? input.fileHandleForWriting.close() }75 let data = out.fileHandleForReading.readDataToEndOfFile()76 process.waitUntilExit()77 guard process.terminationStatus == 0 else {78 let message = (try? String(contentsOf: errorURL, encoding: .utf8)) ?? ""79 throw Failure(description: "\(tool) \(arguments.joined(separator: " ")) → \(process.terminationStatus): \(message.prefix(400))")80 }81 return data82 }8384 static func works(_ tool: String, _ arguments: [String], cwd: URL? = nil) -> Bool { (try? run(tool, arguments, cwd: cwd)) != nil }85}8687enum Tree {88 static let baseTime = 1_614_834_366 // 2021-03-04 05:06:06 UTC, an even second (ZIP's DOS clock)8990 static func bigLog(megabytes: Int = 8) -> Data {91 var data = Data(capacity: megabytes << 20)92 var line = 093 while data.count < megabytes << 20 { data.append(Data("2026-11-21T10:\(line % 60):00 INFO request \(line) served in \(line % 97) ms by worker \(line % 8)\n".utf8)); line += 1 }94 return data95 }9697 static func random(_ count: Int, seed: UInt64 = 1) -> Data {98 var state = seed99 var data = Data(capacity: count)100 for _ in 0..<count { state = state &* 6364136223846793005 &+ 1442695040888963407; data.append(UInt8(truncatingIfNeeded: state >> 56)) }101 return data102 }103104 /// Unicode names (precomposed, decomposed, CJK, emoji), spaces, an empty folder, relative symbolic links,105 /// executable and private modes, a large compressible file and a zero-byte file.106 @discardableResult107 static func make(at root: URL, bigMegabytes: Int = 8) throws -> URL {108 let fm = FileManager.default109 try fm.createDirectory(at: root.appendingPathComponent("dir with spaces/nested/deep"), withIntermediateDirectories: true)110 try fm.createDirectory(at: root.appendingPathComponent("empty-dir"), withIntermediateDirectories: true)111 try Data("Hello, archive!\n".utf8).write(to: root.appendingPathComponent("hello.txt"))112 try Data().write(to: root.appendingPathComponent("empty.bin"))113 try Data("#!/bin/sh\necho run\n".utf8).write(to: root.appendingPathComponent("run.sh"))114 try Data("secret\n".utf8).write(to: root.appendingPathComponent("private.txt"))115 try Data("unicode\n".utf8).write(to: root.appendingPathComponent("Ünïcødé – 日本語 – 🙂.txt"))116 try Data("decomposed\n".utf8).write(to: root.appendingPathComponent("nfd-e\u{0301}.txt"))117 try random(4096).write(to: root.appendingPathComponent("dir with spaces/nested/deep/file.dat"))118 try bigLog(megabytes: bigMegabytes).write(to: root.appendingPathComponent("big.log"))119 try fm.createSymbolicLink(atPath: root.appendingPathComponent("link-to-hello").path, withDestinationPath: "hello.txt")120 try fm.createSymbolicLink(atPath: root.appendingPathComponent("dir with spaces/up-link").path, withDestinationPath: "../hello.txt")121 chmod(root.appendingPathComponent("run.sh").path, 0o755)122 chmod(root.appendingPathComponent("private.txt").path, 0o600)123 chmod(root.appendingPathComponent("empty-dir").path, 0o750)124 // Distinct, even modification times; folders last (creating children touches them).125 var n = 0126 let all = try snapshot(root).keys.sorted().reversed()127 for relative in all {128 let path = root.appendingPathComponent(relative).path129 var info = stat()130 guard lstat(path, &info) == 0, info.st_mode & S_IFMT != S_IFLNK else { continue }131 var times = [timeval(tv_sec: baseTime + n * 2, tv_usec: 0), timeval(tv_sec: baseTime + n * 2, tv_usec: 0)]132 utimes(path, ×)133 n += 1134 }135 return root136 }137138 struct Item: Equatable, CustomStringConvertible {139 var kind: String140 var mode: mode_t141 var size: Int64142 var crc: UInt32143 var target: String144 var mtime: Int145 var description: String { "\(kind) \(String(mode, radix: 8)) \(size)b crc=\(String(crc, radix: 16)) \(target) t=\(mtime)" }146 }147148 /// Relative path → what is there (links are not followed).149 static func snapshot(_ root: URL) throws -> [String: Item] {150 var out: [String: Item] = [:]151 let base = root.resolvingSymlinksInPath().path152 guard let enumerator = FileManager.default.enumerator(atPath: base) else { return out }153 while let relative = enumerator.nextObject() as? String {154 let path = base + "/" + relative155 var info = stat()156 guard lstat(path, &info) == 0 else { continue }157 var item = Item(kind: "other", mode: info.st_mode & 0o7777, size: 0, crc: 0, target: "", mtime: info.st_mtimespec.tv_sec)158 switch info.st_mode & S_IFMT {159 case S_IFREG:160 item.kind = "file"; item.size = Int64(info.st_size)161 let data = try Data(contentsOf: URL(fileURLWithPath: path), options: .mappedIfSafe)162 item.crc = data.withUnsafeBytes { UInt32(truncatingIfNeeded: crc32(0, $0.bindMemory(to: Bytef.self).baseAddress, uInt(data.count))) }163 case S_IFDIR: item.kind = "dir"164 case S_IFLNK:165 item.kind = "link"; item.mode = 0; item.mtime = 0166 item.target = (try? FileManager.default.destinationOfSymbolicLink(atPath: path)) ?? "?"167 default: break168 }169 out[relative] = item170 }171 return out172 }173174 /// Compares two trees. `times` and `modes` can be relaxed for tools that do not carry them.175 static func expectEqual(_ a: URL, _ b: URL, times: Bool = true, modes: Bool = true, directoryTimes: Bool = true, _ label: String, sourceLocation: SourceLocation = #_sourceLocation) throws {176 let x = try snapshot(a), y = try snapshot(b)177 #expect(Set(x.keys) == Set(y.keys), "\(label): names differ — only in first: \(Set(x.keys).subtracting(y.keys).sorted()), only in second: \(Set(y.keys).subtracting(x.keys).sorted())", sourceLocation: sourceLocation)178 for (name, var item) in x {179 guard var other = y[name] else { continue }180 if !times || (item.kind == "dir" && !directoryTimes) { item.mtime = 0; other.mtime = 0 }181 if !modes { item.mode = 0; other.mode = 0 }182 #expect(item == other, "\(label): “\(name)” — \(item) vs \(other)", sourceLocation: sourceLocation)183 }184 }185}186187struct Converted {188 let output: ConversionOutput189 let collector: Collector190 let checks: [ValidationCheck]191 let work: URL192 var url: URL { output.files[0] }193}194195/// Runs a request through the engine, then the core's generic validation and the engine's own.196func runEngine(_ sources: [URL], _ from: FormatID, _ to: FormatID, operation: String?, options: [String: OptionValue] = [:], name: String? = nil,197 engine: ArchiveEngine = ArchiveEngine(), in scratch: Scratch) async throws -> Converted {198 let work = try scratch.work()199 let collector = Collector()200 let request = ConversionRequest(source: sources[0], sourceFormat: from, destinationFormat: to, operation: operation, options: ConversionOptions(options),201 additionalSources: Array(sources.dropFirst()), outputBaseName: name)202 let output = try await engine.convert(request, context: ConversionContext(workDirectory: work, progress: { collector.add($0) }, warn: { collector.add($0) }))203 var checks = OutputValidator.validate(output, destinationFormat: to, workDirectory: work)204 checks += try await engine.validate(output, request: request, source: nil)205 return Converted(output: output, collector: collector, checks: checks, work: work)206}207208func expectAllPass(_ checks: [ValidationCheck], _ label: String, sourceLocation: SourceLocation = #_sourceLocation) {209 for check in checks { #expect(check.passed, "\(label): \(check.name) — \(check.detail)", sourceLocation: sourceLocation) }210}211212/// Archives made by tools other than this engine.213enum Fixture {214 static func ext(_ f: FormatID) -> String {215 switch f { case .tgz: "tar.gz"; case .tbz2: "tar.bz2"; case .txz: "tar.xz"; case .tzst: "tar.zst"; default: FormatCatalog.format(f)?.preferredExtension ?? f.rawValue }216 }217218 /// Archives `folder` (as top-level entry `folder.lastPathComponent`) with a system tool. Returns `nil` when no219 /// system tool can write the format (then the caller falls back to the engine and says so).220 static func system(_ format: FormatID, of folder: URL, in scratch: Scratch, name: String = "fixture") throws -> URL? {221 let out = scratch.url("\(name)-\(UUID().uuidString.prefix(6)).\(ext(format))")222 let parent = folder.deletingLastPathComponent(), top = folder.lastPathComponent223 switch format {224 case .zip: try Tool.run("/usr/bin/zip", ["-r", "-y", "-q", out.path, top], cwd: parent)225 case .tar: try Tool.run("/usr/bin/tar", ["-cf", out.path, top], cwd: parent)226 case .tgz: try Tool.run("/usr/bin/tar", ["-czf", out.path, top], cwd: parent)227 case .tbz2: try Tool.run("/usr/bin/tar", ["-cjf", out.path, top], cwd: parent)228 case .txz: try Tool.run("/usr/bin/tar", ["-cJf", out.path, top], cwd: parent)229 case .tzst:230 // The system bsdtar has no zstd of its own; it needs a `zstd` program, which only Homebrew provides.231 guard let zstd = homebrew("zstd") else { return nil }232 let tar = scratch.url("\(name)-\(UUID().uuidString.prefix(6)).tar")233 try Tool.run("/usr/bin/tar", ["-cf", tar.path, top], cwd: parent)234 try Tool.run(zstd, ["-q", "-c", tar.path]).write(to: out)235 case .sevenZip: guard Tool.works("/usr/bin/tar", ["--format", "7zip", "-cf", out.path, top], cwd: parent) else { return nil }236 default: return nil237 }238 return out239 }240241 /// A single compressed stream of `file` made by a system tool, or `nil` when none exists for the format.242 static func systemStream(_ format: FormatID, of file: URL, in scratch: Scratch) throws -> URL? {243 let out = scratch.url("\(file.lastPathComponent).\(ext(format))")244 switch format {245 case .gz: try Tool.run("/usr/bin/gzip", ["-c", file.path]).write(to: out)246 case .bz2: try Tool.run("/usr/bin/bzip2", ["-c", file.path]).write(to: out)247 case .xz: guard let tool = homebrew("xz") else { return nil }; try Tool.run(tool, ["-c", file.path]).write(to: out)248 case .zst: guard let tool = homebrew("zstd") else { return nil }; try Tool.run(tool, ["-q", "-c", file.path]).write(to: out)249 default: return nil250 }251 return out252 }253254 /// macOS ships no `xz` or `zstd` command. When Homebrew's happen to be installed they give a decoder and an255 /// encoder that are independent of the bundled libraries; the tests use them if present and say so otherwise.256 static func homebrew(_ name: String) -> String? {257 let path = "/opt/homebrew/bin/" + name258 return FileManager.default.isExecutableFile(atPath: path) ? path : nil259 }260261 /// Decompresses a single stream with a tool that is not this engine, or `nil` when no such tool exists.262 static func systemDecompress(_ archive: URL, _ format: FormatID) throws -> Data? {263 switch format {264 case .gz: return try Tool.run("/usr/bin/gzip", ["-dc", archive.path])265 case .bz2: return try Tool.run("/usr/bin/bzip2", ["-dc", archive.path])266 case .xz: return try homebrew("xz").map { try Tool.run($0, ["-dc", archive.path]) }267 case .zst: return try homebrew("zstd").map { try Tool.run($0, ["-q", "-dc", archive.path]) }268 default: return nil269 }270 }271272 /// `true` when a tool other than this engine can read the container format on this Mac.273 static func systemReads(_ format: FormatID) -> Bool { format != .tzst || homebrew("zstd") != nil }274275 /// Extracts with a system tool into a new folder.276 static func systemExtract(_ archive: URL, _ format: FormatID, in scratch: Scratch, password: String? = nil) throws -> URL {277 let out = try scratch.directory("sys-out-\(UUID().uuidString.prefix(6))")278 if format == .zip, password == nil { try Tool.run("/usr/bin/unzip", ["-q", archive.path, "-d", out.path]) }279 else if format == .tzst, let zstd = homebrew("zstd") {280 let tar = scratch.url("sys-\(UUID().uuidString.prefix(6)).tar")281 try Tool.run(zstd, ["-q", "-dc", archive.path]).write(to: tar)282 try Tool.run("/usr/bin/tar", ["-xpf", tar.path, "-C", out.path])283 } else { try Tool.run("/usr/bin/tar", (password.map { ["--passphrase", $0] } ?? []) + ["-xpf", archive.path, "-C", out.path]) }284 return out285 }286}287288/// Byte-exact archive builders, for inputs no well-behaved tool would write.289enum Craft {290 static func le16(_ v: Int) -> [UInt8] { [UInt8(v & 0xFF), UInt8(v >> 8 & 0xFF)] }291 static func le32(_ v: UInt32) -> [UInt8] { [UInt8(v & 0xFF), UInt8(v >> 8 & 0xFF), UInt8(v >> 16 & 0xFF), UInt8(v >> 24)] }292 static func crc(_ bytes: [UInt8]) -> UInt32 { UInt32(truncatingIfNeeded: crc32(0, bytes, uInt(bytes.count))) }293294 struct TarEntry {295 var name: [UInt8]296 var type: UInt8 = UInt8(ascii: "0") // 0 file, 1 hard link, 2 symlink, 3 char device, 4 block device, 5 directory, 6 FIFO297 var mode = 0o644298 var data: [UInt8] = []299 var link: [UInt8] = []300 init(_ name: String, type: Character = "0", mode: Int = 0o644, data: String = "", link: String = "") {301 self.name = Array(name.utf8); self.type = type.asciiValue ?? 48; self.mode = mode; self.data = Array(data.utf8); self.link = Array(link.utf8)302 }303 }304305 /// ustar. Names up to 100 bytes go in the header; longer ones use a GNU `L` long-name record.306 static func tar(_ entries: [TarEntry]) -> Data {307 func header(name: [UInt8], type: UInt8, mode: Int, size: Int, link: [UInt8]) -> [UInt8] {308 var h = [UInt8](repeating: 0, count: 512)309 func put(_ bytes: [UInt8], _ at: Int, _ max: Int) { for (i, b) in bytes.prefix(max).enumerated() { h[at + i] = b } }310 func octal(_ v: Int, _ at: Int, _ width: Int) { put(Array(String(v, radix: 8).leftPadded(width - 1).utf8) + [0], at, width) }311 put(name, 0, 100); octal(mode, 100, 8); octal(0, 108, 8); octal(0, 116, 8); octal(size, 124, 12); octal(Tree.baseTime, 136, 12)312 put([UInt8](repeating: 0x20, count: 8), 148, 8)313 h[156] = type314 put(link, 157, 100); put(Array("ustar".utf8) + [0], 257, 6); put(Array("00".utf8), 263, 2)315 let sum = h.reduce(0) { $0 + Int($1) }316 put(Array(String(sum, radix: 8).leftPadded(6).utf8) + [0, 0x20], 148, 8)317 return h318 }319 func padded(_ bytes: [UInt8]) -> [UInt8] { bytes + [UInt8](repeating: 0, count: (512 - bytes.count % 512) % 512) }320 var out: [UInt8] = []321 for e in entries {322 if e.name.count > 100 {323 out += header(name: Array("././@LongLink".utf8), type: UInt8(ascii: "L"), mode: 0, size: e.name.count + 1, link: [])324 out += padded(e.name + [0])325 }326 if e.link.count > 100 {327 out += header(name: Array("././@LongLink".utf8), type: UInt8(ascii: "K"), mode: 0, size: e.link.count + 1, link: [])328 out += padded(e.link + [0])329 }330 let hasData = e.type == UInt8(ascii: "0")331 out += header(name: e.name, type: e.type, mode: e.mode, size: hasData ? e.data.count : 0, link: e.link)332 if hasData { out += padded(e.data) }333 }334 out += [UInt8](repeating: 0, count: 1024)335 return Data(out)336 }337338 struct ZipEntry {339 var name: [UInt8]340 var data: [UInt8]341 /// Unix mode with file type (0o100644 file, 0o120777 symlink, 0o040755 directory); 0 = DOS attributes only.342 var unixMode: UInt32 = 0o100644343 var utf8 = true344 init(_ name: String, _ data: String = "", unixMode: UInt32 = 0o100644) { self.name = Array(name.utf8); self.data = Array(data.utf8); self.unixMode = unixMode }345 init(rawName: [UInt8], data: String, utf8: Bool = false) { name = rawName; self.data = Array(data.utf8); self.utf8 = utf8 }346 }347348 /// ZIP with stored (uncompressed) entries, exactly as given: names are not checked in any way.349 static func zip(_ entries: [ZipEntry], comment: String = "") -> Data {350 var out: [UInt8] = [], central: [UInt8] = []351 for e in entries {352 let offset = UInt32(out.count), sum = crc(e.data), flags = e.utf8 ? 0x800 : 0353 var common: [UInt8] = le16(flags)354 common += le16(0); common += le16(0x6000); common += le16(0x5264); common += le32(sum)355 common += le32(UInt32(e.data.count)); common += le32(UInt32(e.data.count)); common += le16(e.name.count); common += le16(0)356 out += le32(0x0403_4b50); out += le16(20); out += common; out += e.name; out += e.data357 central += le32(0x0201_4b50); central += le16(0x0314); central += le16(20); central += common358 central += le16(0); central += le16(0); central += le16(0); central += le32(e.unixMode << 16); central += le32(offset); central += e.name359 }360 let start = UInt32(out.count)361 out += central362 out += le32(0x0605_4b50); out += le16(0); out += le16(0); out += le16(entries.count); out += le16(entries.count)363 out += le32(UInt32(central.count)); out += le32(start); out += le16(comment.utf8.count); out += Array(comment.utf8)364 return Data(out)365 }366367 /// RAR 4 (format 1.5–4.x) with stored files, assembled from the published format description:368 /// marker block, archive header (0x73), one file header (0x74, method 0x30 = store) per file, end block (0x7B).369 /// Header CRCs are the low 16 bits of CRC-32 over the header after its CRC field.370 static func rar4(_ files: [(name: String, data: String)]) -> Data {371 func block(_ body: [UInt8]) -> [UInt8] { le16(Int(crc(body) & 0xFFFF)) + body }372 var out: [UInt8] = [0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x00]373 out += block([0x73] + le16(0) + le16(13) + le16(0) + le32(0))374 for file in files {375 let name = Array(file.name.utf8), data = Array(file.data.utf8)376 // DOS date/time 2021-03-04 05:06:06.377 let date: UInt32 = (2021 - 1980) << 25 | 3 << 21 | 4 << 16378 let time: UInt32 = 5 << 11 | 6 << 5 | 3379 var body: [UInt8] = [0x74]380 body += le16(0x8000); body += le16(32 + name.count)381 body += le32(UInt32(data.count)); body += le32(UInt32(data.count))382 body += [3]; body += le32(crc(data)); body += le32(date | time)383 body += [20, 0x30]; body += le16(name.count); body += le32(0o100644); body += name384 out += block(body) + data385 }386 out += block([0x7B] + le16(0x4000) + le16(7))387 return Data(out)388 }389390 static func vint(_ value: Int) -> [UInt8] {391 var v = value, out: [UInt8] = []392 repeat { var b = UInt8(v & 0x7F); v >>= 7; if v > 0 { b |= 0x80 }; out.append(b) } while v > 0393 return out394 }395396 /// RAR 5 with stored files, assembled from the published format description: 8-byte signature, main archive397 /// header (type 1), one file header (type 2, compression method 0) per file followed by its data area, end of398 /// archive header (type 5). Each header is `CRC-32 | size (vint) | type | flags | fields…`, the CRC covering399 /// everything after itself.400 static func rar5(_ files: [(name: String, data: String)]) -> Data {401 func header(_ fields: [UInt8]) -> [UInt8] {402 let sized = vint(fields.count) + fields403 return le32(crc(sized)) + sized404 }405 var out: [UInt8] = [0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x01, 0x00]406 out += header(vint(1) + vint(0) + vint(0))407 for file in files {408 let name = Array(file.name.utf8), data = Array(file.data.utf8)409 // Header flags 0x02: a data area follows. File flags 0x02 | 0x04: Unix time and CRC-32 present.410 var fields = vint(2)411 fields += vint(0x02); fields += vint(data.count)412 fields += vint(0x02 | 0x04); fields += vint(data.count); fields += vint(0o100644)413 fields += le32(UInt32(Tree.baseTime)); fields += le32(crc(data))414 fields += vint(0); fields += vint(1); fields += vint(name.count); fields += name // compression info 0 = stored, host OS 1 = Unix415 out += header(fields) + data416 }417 out += header(vint(5) + vint(0) + vint(0))418 return Data(out)419 }420}421422extension String {423 func leftPadded(_ width: Int) -> String { String(repeating: "0", count: max(0, width - count)) + self }424}425