Repository Public License
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Convert9// File : Packages/EngineArchive/Tests/EngineArchiveTests/RoundTripTests.swift10// Module : EngineArchiveTests11// Repository : https://git.spboucher.ai/orix-convert12// Created : 2026-11-2113// Last Modified : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author : Simon-Pierre Boucher20// Position : Professor of Finance21// Institution : Université du Québec en Outaouais (UQO)22// Department : Département des sciences administratives23// Organization : SP Boucher24//25// Website : https://spboucher.ai26// Git : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General : info@spboucher.ai33// Professional : contact@spboucher.ai34// Direct : simon@spboucher.ai35// Research : research@spboucher.ai36// Development : dev@spboucher.ai37// Support : support@spboucher.ai38// Security/Admin : admin@spboucher.ai39// Notifications : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification : Public66// Distribution : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License : Proprietary, all rights reserved (see license file)80// License File : LICENSE81// Maintainer : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788// ORIX Convert — Packages/EngineArchive/Tests/EngineArchiveTests/RoundTripTests.swift89// Description: Round trips for every format the engine writes, archives made by other tools read by the engine,90// the engine's archives read by other tools, reproducible output, options and encryption.91// Author: Simon-Pierre Boucher — contact@spboucher.ai92import ConvertCore93@testable import EngineArchive94import Foundation95import Testing9697@Suite struct RoundTripTests {98 static let create = ArchiveEngine.createOperation, extract = ArchiveEngine.extractOperation99100 @Test(arguments: ArchiveKind.containers)101 func createThenExtractGivesBackTheTree(format: FormatID) async throws {102 let scratch = try Scratch(); defer { scratch.cleanUp() }103 let tree = try Tree.make(at: scratch.url("tree"))104 let archive = try await runEngine([tree], .folder, format, operation: Self.create, in: scratch)105 expectAllPass(archive.checks, "create \(format)")106 #expect(FileTypeDetector.detect(archive.url).format == format)107 #expect(archive.output.facts["files"] == "8" && archive.output.facts["links"] == "2" && archive.output.facts["folders"] == "5")108 #expect(archive.collector.progress.last == 1 && archive.collector.progress == archive.collector.progress.sorted())109 #expect(archive.collector.warnings.isEmpty, "\(archive.collector.warnings)")110 let size = try #require(try archive.url.resourceValues(forKeys: [.fileSizeKey]).fileSize)111 if format != .tar { #expect(size < 2 << 20, "8 MB of log lines should compress well, got \(size)") }112113 let out = try await runEngine([archive.url], format, .folder, operation: Self.extract, name: "out", in: scratch)114 expectAllPass(out.checks, "extract \(format)")115 try Tree.expectEqual(tree, out.url.appendingPathComponent("tree"), "engine → engine \(format)")116 #expect(out.collector.warnings.isEmpty, "\(out.collector.warnings)")117118 // The same archive read by a tool that is not this engine (none reads zstd on a stock macOS).119 if Fixture.systemReads(format) {120 let system = try Fixture.systemExtract(archive.url, format, in: scratch)121 try Tree.expectEqual(tree, system.appendingPathComponent("tree"), directoryTimes: format != .zip, "engine → system tool \(format)")122 }123 }124125 @Test(arguments: ArchiveKind.containers)126 func archivesMadeByOtherToolsExtractIdentically(format: FormatID) async throws {127 let scratch = try Scratch(); defer { scratch.cleanUp() }128 let tree = try Tree.make(at: scratch.url("tree"), bigMegabytes: 2)129 guard let fixture = try Fixture.system(format, of: tree, in: scratch) else {130 #expect(format == .tzst, "no tool writes \(format) on this Mac") // only zstd can lack an outside encoder131 return132 }133 let out = try await runEngine([fixture], format, .folder, operation: Self.extract, name: "out", in: scratch)134 expectAllPass(out.checks, "extract system \(format)")135 // bsdtar's 7z writer does not store directory dates the way it stores file dates: compared for files only.136 try Tree.expectEqual(tree, out.url.appendingPathComponent("tree"), directoryTimes: format != .sevenZip, "system tool → engine \(format)")137 #expect(out.output.facts["files"] == "8")138 }139140 @Test func dittoArchivesLoseOnlyTheirHousekeepingEntries() async throws {141 let scratch = try Scratch(); defer { scratch.cleanUp() }142 let tree = try Tree.make(at: scratch.url("tree"), bigMegabytes: 1)143 // An extended attribute makes ditto add __MACOSX/._ companions, as Finder's "Compress" does.144 _ = Data("yes".utf8).withUnsafeBytes { setxattr(tree.appendingPathComponent("hello.txt").path, "ai.spboucher.test", $0.baseAddress, 3, 0, 0) }145 try Data("ds".utf8).write(to: tree.appendingPathComponent(".DS_Store"))146 let zip = scratch.url("ditto.zip")147 try Tool.run("/usr/bin/ditto", ["-c", "-k", "--sequesterRsrc", "--keepParent", tree.path, zip.path])148 let listing = try await ArchiveBrowser.list(zip, format: .zip)149 #expect(listing.entries.contains { $0.path.hasPrefix("__MACOSX/") })150 let out = try await runEngine([zip], .zip, .folder, operation: Self.extract, name: "out", in: scratch)151 expectAllPass(out.checks, "ditto")152 #expect(out.collector.message("macos-files-excluded").contains("macOS housekeeping item"))153 #expect(!FileManager.default.fileExists(atPath: out.url.appendingPathComponent("__MACOSX").path))154 try FileManager.default.removeItem(at: tree.appendingPathComponent(".DS_Store"))155 try Tree.expectEqual(tree, out.url.appendingPathComponent("tree"), directoryTimes: false, "ditto → engine")156 // Asked to keep them, the engine keeps them.157 let all = try await runEngine([zip], .zip, .folder, operation: Self.extract, options: ["exclude.macos": .bool(false)], name: "all", in: scratch)158 #expect(FileManager.default.fileExists(atPath: all.url.appendingPathComponent("__MACOSX/tree/._hello.txt").path))159 #expect(FileManager.default.fileExists(atPath: all.url.appendingPathComponent("tree/.DS_Store").path) && !all.collector.has("macos-files-excluded"))160 }161162 @Test(arguments: ArchiveKind.containers)163 func reproducibleArchivesAreByteIdentical(format: FormatID) async throws {164 let scratch = try Scratch(); defer { scratch.cleanUp() }165 let a = try Tree.make(at: scratch.url("one/tree"), bigMegabytes: 1)166 // Same content created separately: other dates, other creation order, another umask-like mode.167 let b = try Tree.make(at: scratch.url("two/tree"), bigMegabytes: 1)168 _ = try Tool.run("/usr/bin/touch", [b.appendingPathComponent("hello.txt").path, b.appendingPathComponent("dir with spaces").path])169 chmod(b.appendingPathComponent("hello.txt").path, 0o664)170 let options: [String: OptionValue] = ["deterministic": .bool(true)]171 let first = try await runEngine([a], .folder, format, operation: Self.create, options: options, in: scratch)172 let second = try await runEngine([b], .folder, format, operation: Self.create, options: options, in: scratch)173 expectAllPass(first.checks, "deterministic \(format)")174 #expect(try Data(contentsOf: first.url) == Data(contentsOf: second.url))175 // Without the option the two archives differ (dates), which shows the option is what makes them equal.176 let plainA = try await runEngine([a], .folder, format, operation: Self.create, in: scratch)177 let plainB = try await runEngine([b], .folder, format, operation: Self.create, in: scratch)178 #expect(try Data(contentsOf: plainA.url) != Data(contentsOf: plainB.url))179 // Dates and modes are the fixed ones.180 let out = try await runEngine([first.url], format, .folder, operation: Self.extract, name: "out", in: scratch)181 let items = try Tree.snapshot(out.url.appendingPathComponent("tree"))182 #expect(items["hello.txt"]?.mtime == ArchiveSettings.deterministicTime && items["hello.txt"]?.mode == 0o644 && items["run.sh"]?.mode == 0o755 && items["private.txt"]?.mode == 0o644)183 }184185 @Test func entriesAreStoredInSortedOrderWithParentsFirst() async throws {186 let scratch = try Scratch(); defer { scratch.cleanUp() }187 let tree = try Tree.make(at: scratch.url("tree"), bigMegabytes: 1)188 let archive = try await runEngine([tree], .folder, .tar, operation: Self.create, in: scratch)189 let names = try await ArchiveBrowser.list(archive.url, format: .tar).entries.map(\.path)190 #expect(names.first == "tree/" || names.first == "tree")191 let cleaned = names.map { $0.hasSuffix("/") ? String($0.dropLast()) : $0 }192 for (i, name) in cleaned.enumerated() where name.contains("/") {193 let parent = String(name[..<(name.lastIndex(of: "/") ?? name.startIndex)])194 #expect(cleaned[..<i].contains(parent), "\(name) comes before its folder")195 }196 #expect(cleaned.contains("tree/empty-dir") && cleaned.contains("tree/Ünïcødé – 日本語 – 🙂.txt") && cleaned.contains("tree/nfd-e\u{0301}.txt"))197 }198199 @Test func contentOptionsForCreation() async throws {200 let scratch = try Scratch(); defer { scratch.cleanUp() }201 let tree = try Tree.make(at: scratch.url("tree"), bigMegabytes: 1)202 try Data("x".utf8).write(to: tree.appendingPathComponent(".DS_Store"))203 try Data("x".utf8).write(to: tree.appendingPathComponent("._hello.txt"))204 try FileManager.default.createDirectory(at: tree.appendingPathComponent("__MACOSX"), withIntermediateDirectories: true)205 try Data("x".utf8).write(to: tree.appendingPathComponent("__MACOSX/junk"))206 // Default: housekeeping left out and reported; the folder is the top level.207 let normal = try await runEngine([tree], .folder, .zip, operation: Self.create, in: scratch)208 var names = try await ArchiveBrowser.list(normal.url, format: .zip).entries.map(\.path)209 #expect(!names.contains { $0.contains(".DS_Store") || $0.contains("__MACOSX") || $0.contains("._") } && names.allSatisfy { $0.hasPrefix("tree") })210 #expect(normal.collector.message("macos-files-excluded").hasPrefix("3 macOS housekeeping items"), "\((try? FileManager.default.contentsOfDirectory(atPath: tree.path).sorted()) ?? [])")211 // Kept on request; contents at the root on request.212 let everything = try await runEngine([tree], .folder, .zip, operation: Self.create, options: ["exclude.macos": .bool(false), "root.keep": .bool(false)], in: scratch)213 names = try await ArchiveBrowser.list(everything.url, format: .zip).entries.map(\.path)214 #expect(names.contains(".DS_Store") && names.contains("__MACOSX/junk") && names.contains("hello.txt") && !names.contains { $0.hasPrefix("tree") })215 expectAllPass(everything.checks, "everything")216 // Several sources: files and folders side by side, same names kept apart.217 let other = try scratch.directory("elsewhere/tree")218 try Data("other".utf8).write(to: other.appendingPathComponent("b.txt"))219 let loose = try scratch.file("loose.txt", Data("loose".utf8))220 let multi = try await runEngine([tree, loose, other], .folder, .tgz, operation: Self.create, in: scratch)221 expectAllPass(multi.checks, "multi")222 names = try await ArchiveBrowser.list(multi.url, format: .tgz).entries.map(\.path)223 #expect(names.contains("loose.txt") && names.contains("tree/hello.txt") && names.contains("tree 2/b.txt"))224 // Compression levels: store is larger than maximum, and every level reads back.225 var sizes: [String: Int] = [:]226 for level in ["store", "fast", "normal", "maximum"] {227 for format in [FormatID.zip, .sevenZip] {228 let a = try await runEngine([tree], .folder, format, operation: Self.create, options: ["compression.level": .string(level)], in: scratch)229 expectAllPass(a.checks, "\(format) \(level)")230 sizes["\(format)-\(level)"] = try a.url.resourceValues(forKeys: [.fileSizeKey]).fileSize231 }232 }233 #expect(try #require(sizes["zip-store"]) > 1 << 20 && (sizes["zip-maximum"] ?? 0) < (sizes["zip-fast"] ?? 0) && (sizes["7z-maximum"] ?? .max) < (sizes["7z-store"] ?? 0) / 4)234 await #expect(throws: ConversionError.self) { _ = try await runEngine([tree], .folder, .tgz, operation: Self.create, options: ["compression.level": .string("store")], in: scratch) }235 // Nothing to archive is an error, not an empty archive.236 let empty = try scratch.directory("empty/only-noise")237 try Data("x".utf8).write(to: empty.appendingPathComponent(".DS_Store"))238 await #expect(throws: ConversionError.self) { _ = try await runEngine([empty], .folder, .zip, operation: Self.create, options: ["root.keep": .bool(false)], in: scratch) }239 }240241 @Test func extendedAttributesTravelInTarWhenAsked() async throws {242 let scratch = try Scratch(); defer { scratch.cleanUp() }243 let tree = try Tree.make(at: scratch.url("tree"), bigMegabytes: 1)244 let file = tree.appendingPathComponent("hello.txt").path245 _ = Data("blue".utf8).withUnsafeBytes { setxattr(file, "ai.spboucher.colour", $0.baseAddress, 4, 0, 0) }246 let with = try await runEngine([tree], .folder, .txz, operation: Self.create, options: ["xattrs": .bool(true)], in: scratch)247 let restored = try await runEngine([with.url], .txz, .folder, operation: Self.extract, options: ["xattrs": .bool(true)], name: "r", in: scratch)248 #expect(SecureExtractor.readXattr(restored.url.appendingPathComponent("tree/hello.txt").path, "ai.spboucher.colour") == Data("blue".utf8))249 let ignored = try await runEngine([with.url], .txz, .folder, operation: Self.extract, name: "i", in: scratch)250 #expect(SecureExtractor.readXattr(ignored.url.appendingPathComponent("tree/hello.txt").path, "ai.spboucher.colour") == nil)251 let without = try await runEngine([tree], .folder, .txz, operation: Self.create, in: scratch)252 let plain = try await runEngine([without.url], .txz, .folder, operation: Self.extract, options: ["xattrs": .bool(true)], name: "p", in: scratch)253 #expect(SecureExtractor.readXattr(plain.url.appendingPathComponent("tree/hello.txt").path, "ai.spboucher.colour") == nil)254 // ZIP cannot hold them: said before converting.255 let request = ConversionRequest(source: tree, sourceFormat: .folder, destinationFormat: .zip, operation: Self.create, options: ConversionOptions(["xattrs": .bool(true)]))256 #expect(ArchiveEngine().preflight(request, inspection: nil).contains { $0.code == "xattrs-unsupported" })257 }258259 @Test(arguments: ["aes256", "zipcrypto"])260 func encryptedZipRoundTripsAndRefusesWrongPasswords(method: String) async throws {261 let scratch = try Scratch(); defer { scratch.cleanUp() }262 let tree = try Tree.make(at: scratch.url("tree"), bigMegabytes: 1)263 let password = "correct horse — バッテリー"264 let archive = try await runEngine([tree], .folder, .zip, operation: Self.create, options: ["password": .string(password), "encrypt": .bool(true), "encryption": .string(method)], in: scratch)265 expectAllPass(archive.checks, "encrypted \(method)")266 #expect(archive.checks.contains { $0.name == "Every file is encrypted" && $0.passed })267 // The content is really unreadable without the password: the log lines are nowhere in the file.268 let raw = try Data(contentsOf: archive.url)269 #expect(raw.range(of: Data("Hello, archive!".utf8)) == nil && raw.range(of: Data("INFO request".utf8)) == nil)270 let page = try await ArchiveBrowser.list(archive.url, format: .zip)271 #expect(page.summary.hasEncryptedEntries && page.entries.contains { $0.isEncrypted && $0.path == "tree/hello.txt" })272 let inspection = try await ArchiveEngine().inspect(archive.url, format: .zip)273 #expect(inspection.isEncrypted == true)274275 let good = try await runEngine([archive.url], .zip, .folder, operation: Self.extract, options: ["password": .string(password)], name: "good", in: scratch)276 expectAllPass(good.checks, "decrypt \(method)")277 try Tree.expectEqual(tree, good.url.appendingPathComponent("tree"), "decrypted \(method)")278 for (wrong, expected) in [("nope", "not the right one"), ("", "protected by a password")] {279 let work = try scratch.work()280 let request = ConversionRequest(source: archive.url, sourceFormat: .zip, destinationFormat: .folder, operation: Self.extract, options: ConversionOptions(["password": .string(wrong)]), outputBaseName: "bad")281 do {282 _ = try await ArchiveEngine().convert(request, context: .init(workDirectory: work))283 Issue.record("extracted with password “\(wrong)”")284 } catch ConversionError.encryptedInput(let message) {285 #expect(message.contains(expected), "\(message)")286 #expect(!message.contains(password))287 }288 #expect(try FileManager.default.contentsOfDirectory(atPath: work.path).isEmpty, "partial output left behind")289 let warnings = ArchiveEngine().preflight(request, inspection: inspection)290 #expect(warnings.contains { $0.code == "password-needed" } == wrong.isEmpty)291 }292 // An independent implementation opens an archive made the same way. (ASCII password: the test harness hands293 // arguments to the tool in decomposed Unicode, which would not be the bytes the engine used.)294 let ascii = try await runEngine([tree], .folder, .zip, operation: Self.create, options: ["password": .string("Tr0ub4dor&3"), "encrypt": .bool(true), "encryption": .string(method)], in: scratch)295 let system = try Fixture.systemExtract(ascii.url, .zip, in: scratch, password: "Tr0ub4dor&3")296 try Tree.expectEqual(tree, system.appendingPathComponent("tree"), directoryTimes: false, "engine → bsdtar, \(method)")297 // Conversion to another format decrypts with the password; the result is not encrypted.298 let converted = try await runEngine([archive.url], .zip, .tzst, operation: nil, options: ["password": .string(password)], in: scratch)299 expectAllPass(converted.checks, "encrypted zip → tzst")300 #expect(try await ArchiveBrowser.list(converted.url, format: .tzst).summary.hasEncryptedEntries == false)301 let verified = try await ArchiveVerifier.verify(archive.url, format: .zip, password: password)302 #expect(verified.passed && verified.checkedEntries == 8)303 await #expect(throws: ConversionError.self) { _ = try await ArchiveVerifier.verify(archive.url, format: .zip, password: "wrong") }304 }305306 @Test func zipCryptoArchiveMadeByTheSystemZipCommandOpensWithItsPassword() async throws {307 let scratch = try Scratch(); defer { scratch.cleanUp() }308 let tree = try Tree.make(at: scratch.url("tree"), bigMegabytes: 1)309 let zip = scratch.url("system-encrypted.zip")310 try Tool.run("/usr/bin/zip", ["-r", "-y", "-q", "-P", "s3cret", zip.path, "tree"], cwd: tree.deletingLastPathComponent())311 let out = try await runEngine([zip], .zip, .folder, operation: Self.extract, options: ["password": .string("s3cret")], name: "out", in: scratch)312 expectAllPass(out.checks, "system zipcrypto")313 // Info-ZIP encrypts the targets of symbolic links too; the decoder cannot decipher those, so the two links314 // of the tree are left out — and reported — rather than created with garbage targets.315 #expect(out.collector.message("symlinks-encrypted").hasPrefix("2 symbolic links have an encrypted target"))316 try FileManager.default.removeItem(at: tree.appendingPathComponent("link-to-hello"))317 try FileManager.default.removeItem(at: tree.appendingPathComponent("dir with spaces/up-link"))318 try Tree.expectEqual(tree, out.url.appendingPathComponent("tree"), directoryTimes: false, "zip -P → engine")319 #expect(try await ArchiveBrowser.list(zip, format: .zip).entries.filter { $0.kind == .symlink }.allSatisfy { $0.linkTarget == nil && $0.isEncrypted })320 await #expect(throws: ConversionError.self) { _ = try await runEngine([zip], .zip, .folder, operation: Self.extract, options: ["password": .string("wrong")], name: "bad", in: scratch) }321 }322323 @Test func encryptionOptionsAreChecked() async throws {324 let scratch = try Scratch(); defer { scratch.cleanUp() }325 let tree = try Tree.make(at: scratch.url("tree"), bigMegabytes: 1)326 func attempt(_ to: FormatID, _ options: [String: OptionValue]) async throws { _ = try await runEngine([tree], .folder, to, operation: Self.create, options: options, in: scratch) }327 await #expect(throws: ConversionError.invalidOption("Encryption needs a password.")) { try await attempt(.zip, ["encrypt": .bool(true)]) }328 await #expect(throws: ConversionError.invalidOption("Only ZIP archives can be encrypted by this app.")) { try await attempt(.sevenZip, ["encrypt": .bool(true), "password": .string("x")]) }329 await #expect(throws: ConversionError.self) { try await attempt(.zip, ["encrypt": .bool(true), "password": .string("x"), "encryption": .string("rot13")]) }330 await #expect(throws: ConversionError.self) { try await attempt(.zip, ["compression.level": .string("ultra")]) }331 await #expect(throws: ConversionError.self) { try await attempt(.zip, ["limit.ratio": .int(-1)]) }332 // A password without "encrypt" does not encrypt (it is only for reading).333 let plain = try await runEngine([tree], .folder, .zip, operation: Self.create, options: ["password": .string("unused")], in: scratch)334 #expect(try await ArchiveBrowser.list(plain.url, format: .zip).summary.hasEncryptedEntries == false)335 let request = ConversionRequest(source: tree, sourceFormat: .folder, destinationFormat: .zip, operation: Self.create,336 options: ConversionOptions(["encrypt": .bool(true), "password": .string("pw"), "encryption": .string("zipcrypto")]))337 let codes = Set(ArchiveEngine().preflight(request, inspection: nil).map(\.code))338 #expect(codes == ["encryption-weak", "encryption-names-visible"])339 }340}341