SPB Git

Repository Public License

RoundTripTests.swift 24.4 KB 341 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Convert9// File             : Packages/EngineArchive/Tests/EngineArchiveTests/RoundTripTests.swift10// Module           : EngineArchiveTests11// Repository       : https://git.spboucher.ai/orix-convert12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788// ORIX Convert — Packages/EngineArchive/Tests/EngineArchiveTests/RoundTripTests.swift89// Description: Round trips for every format the engine writes, archives made by other tools read by the engine,90//              the engine's archives read by other tools, reproducible output, options and encryption.91// Author: Simon-Pierre Boucher — contact@spboucher.ai92import ConvertCore93@testable import EngineArchive94import Foundation95import Testing9697@Suite struct RoundTripTests {98    static let create = ArchiveEngine.createOperation, extract = ArchiveEngine.extractOperation99100    @Test(arguments: ArchiveKind.containers)101    func createThenExtractGivesBackTheTree(format: FormatID) async throws {102        let scratch = try Scratch(); defer { scratch.cleanUp() }103        let tree = try Tree.make(at: scratch.url("tree"))104        let archive = try await runEngine([tree], .folder, format, operation: Self.create, in: scratch)105        expectAllPass(archive.checks, "create \(format)")106        #expect(FileTypeDetector.detect(archive.url).format == format)107        #expect(archive.output.facts["files"] == "8" && archive.output.facts["links"] == "2" && archive.output.facts["folders"] == "5")108        #expect(archive.collector.progress.last == 1 && archive.collector.progress == archive.collector.progress.sorted())109        #expect(archive.collector.warnings.isEmpty, "\(archive.collector.warnings)")110        let size = try #require(try archive.url.resourceValues(forKeys: [.fileSizeKey]).fileSize)111        if format != .tar { #expect(size < 2 << 20, "8 MB of log lines should compress well, got \(size)") }112113        let out = try await runEngine([archive.url], format, .folder, operation: Self.extract, name: "out", in: scratch)114        expectAllPass(out.checks, "extract \(format)")115        try Tree.expectEqual(tree, out.url.appendingPathComponent("tree"), "engine → engine \(format)")116        #expect(out.collector.warnings.isEmpty, "\(out.collector.warnings)")117118        // The same archive read by a tool that is not this engine (none reads zstd on a stock macOS).119        if Fixture.systemReads(format) {120            let system = try Fixture.systemExtract(archive.url, format, in: scratch)121            try Tree.expectEqual(tree, system.appendingPathComponent("tree"), directoryTimes: format != .zip, "engine → system tool \(format)")122        }123    }124125    @Test(arguments: ArchiveKind.containers)126    func archivesMadeByOtherToolsExtractIdentically(format: FormatID) async throws {127        let scratch = try Scratch(); defer { scratch.cleanUp() }128        let tree = try Tree.make(at: scratch.url("tree"), bigMegabytes: 2)129        guard let fixture = try Fixture.system(format, of: tree, in: scratch) else {130            #expect(format == .tzst, "no tool writes \(format) on this Mac")          // only zstd can lack an outside encoder131            return132        }133        let out = try await runEngine([fixture], format, .folder, operation: Self.extract, name: "out", in: scratch)134        expectAllPass(out.checks, "extract system \(format)")135        // bsdtar's 7z writer does not store directory dates the way it stores file dates: compared for files only.136        try Tree.expectEqual(tree, out.url.appendingPathComponent("tree"), directoryTimes: format != .sevenZip, "system tool → engine \(format)")137        #expect(out.output.facts["files"] == "8")138    }139140    @Test func dittoArchivesLoseOnlyTheirHousekeepingEntries() async throws {141        let scratch = try Scratch(); defer { scratch.cleanUp() }142        let tree = try Tree.make(at: scratch.url("tree"), bigMegabytes: 1)143        // An extended attribute makes ditto add __MACOSX/._ companions, as Finder's "Compress" does.144        _ = Data("yes".utf8).withUnsafeBytes { setxattr(tree.appendingPathComponent("hello.txt").path, "ai.spboucher.test", $0.baseAddress, 3, 0, 0) }145        try Data("ds".utf8).write(to: tree.appendingPathComponent(".DS_Store"))146        let zip = scratch.url("ditto.zip")147        try Tool.run("/usr/bin/ditto", ["-c", "-k", "--sequesterRsrc", "--keepParent", tree.path, zip.path])148        let listing = try await ArchiveBrowser.list(zip, format: .zip)149        #expect(listing.entries.contains { $0.path.hasPrefix("__MACOSX/") })150        let out = try await runEngine([zip], .zip, .folder, operation: Self.extract, name: "out", in: scratch)151        expectAllPass(out.checks, "ditto")152        #expect(out.collector.message("macos-files-excluded").contains("macOS housekeeping item"))153        #expect(!FileManager.default.fileExists(atPath: out.url.appendingPathComponent("__MACOSX").path))154        try FileManager.default.removeItem(at: tree.appendingPathComponent(".DS_Store"))155        try Tree.expectEqual(tree, out.url.appendingPathComponent("tree"), directoryTimes: false, "ditto → engine")156        // Asked to keep them, the engine keeps them.157        let all = try await runEngine([zip], .zip, .folder, operation: Self.extract, options: ["exclude.macos": .bool(false)], name: "all", in: scratch)158        #expect(FileManager.default.fileExists(atPath: all.url.appendingPathComponent("__MACOSX/tree/._hello.txt").path))159        #expect(FileManager.default.fileExists(atPath: all.url.appendingPathComponent("tree/.DS_Store").path) && !all.collector.has("macos-files-excluded"))160    }161162    @Test(arguments: ArchiveKind.containers)163    func reproducibleArchivesAreByteIdentical(format: FormatID) async throws {164        let scratch = try Scratch(); defer { scratch.cleanUp() }165        let a = try Tree.make(at: scratch.url("one/tree"), bigMegabytes: 1)166        // Same content created separately: other dates, other creation order, another umask-like mode.167        let b = try Tree.make(at: scratch.url("two/tree"), bigMegabytes: 1)168        _ = try Tool.run("/usr/bin/touch", [b.appendingPathComponent("hello.txt").path, b.appendingPathComponent("dir with spaces").path])169        chmod(b.appendingPathComponent("hello.txt").path, 0o664)170        let options: [String: OptionValue] = ["deterministic": .bool(true)]171        let first = try await runEngine([a], .folder, format, operation: Self.create, options: options, in: scratch)172        let second = try await runEngine([b], .folder, format, operation: Self.create, options: options, in: scratch)173        expectAllPass(first.checks, "deterministic \(format)")174        #expect(try Data(contentsOf: first.url) == Data(contentsOf: second.url))175        // Without the option the two archives differ (dates), which shows the option is what makes them equal.176        let plainA = try await runEngine([a], .folder, format, operation: Self.create, in: scratch)177        let plainB = try await runEngine([b], .folder, format, operation: Self.create, in: scratch)178        #expect(try Data(contentsOf: plainA.url) != Data(contentsOf: plainB.url))179        // Dates and modes are the fixed ones.180        let out = try await runEngine([first.url], format, .folder, operation: Self.extract, name: "out", in: scratch)181        let items = try Tree.snapshot(out.url.appendingPathComponent("tree"))182        #expect(items["hello.txt"]?.mtime == ArchiveSettings.deterministicTime && items["hello.txt"]?.mode == 0o644 && items["run.sh"]?.mode == 0o755 && items["private.txt"]?.mode == 0o644)183    }184185    @Test func entriesAreStoredInSortedOrderWithParentsFirst() async throws {186        let scratch = try Scratch(); defer { scratch.cleanUp() }187        let tree = try Tree.make(at: scratch.url("tree"), bigMegabytes: 1)188        let archive = try await runEngine([tree], .folder, .tar, operation: Self.create, in: scratch)189        let names = try await ArchiveBrowser.list(archive.url, format: .tar).entries.map(\.path)190        #expect(names.first == "tree/" || names.first == "tree")191        let cleaned = names.map { $0.hasSuffix("/") ? String($0.dropLast()) : $0 }192        for (i, name) in cleaned.enumerated() where name.contains("/") {193            let parent = String(name[..<(name.lastIndex(of: "/") ?? name.startIndex)])194            #expect(cleaned[..<i].contains(parent), "\(name) comes before its folder")195        }196        #expect(cleaned.contains("tree/empty-dir") && cleaned.contains("tree/Ünïcødé – 日本語 – 🙂.txt") && cleaned.contains("tree/nfd-e\u{0301}.txt"))197    }198199    @Test func contentOptionsForCreation() async throws {200        let scratch = try Scratch(); defer { scratch.cleanUp() }201        let tree = try Tree.make(at: scratch.url("tree"), bigMegabytes: 1)202        try Data("x".utf8).write(to: tree.appendingPathComponent(".DS_Store"))203        try Data("x".utf8).write(to: tree.appendingPathComponent("._hello.txt"))204        try FileManager.default.createDirectory(at: tree.appendingPathComponent("__MACOSX"), withIntermediateDirectories: true)205        try Data("x".utf8).write(to: tree.appendingPathComponent("__MACOSX/junk"))206        // Default: housekeeping left out and reported; the folder is the top level.207        let normal = try await runEngine([tree], .folder, .zip, operation: Self.create, in: scratch)208        var names = try await ArchiveBrowser.list(normal.url, format: .zip).entries.map(\.path)209        #expect(!names.contains { $0.contains(".DS_Store") || $0.contains("__MACOSX") || $0.contains("._") } && names.allSatisfy { $0.hasPrefix("tree") })210        #expect(normal.collector.message("macos-files-excluded").hasPrefix("3 macOS housekeeping items"), "\((try? FileManager.default.contentsOfDirectory(atPath: tree.path).sorted()) ?? [])")211        // Kept on request; contents at the root on request.212        let everything = try await runEngine([tree], .folder, .zip, operation: Self.create, options: ["exclude.macos": .bool(false), "root.keep": .bool(false)], in: scratch)213        names = try await ArchiveBrowser.list(everything.url, format: .zip).entries.map(\.path)214        #expect(names.contains(".DS_Store") && names.contains("__MACOSX/junk") && names.contains("hello.txt") && !names.contains { $0.hasPrefix("tree") })215        expectAllPass(everything.checks, "everything")216        // Several sources: files and folders side by side, same names kept apart.217        let other = try scratch.directory("elsewhere/tree")218        try Data("other".utf8).write(to: other.appendingPathComponent("b.txt"))219        let loose = try scratch.file("loose.txt", Data("loose".utf8))220        let multi = try await runEngine([tree, loose, other], .folder, .tgz, operation: Self.create, in: scratch)221        expectAllPass(multi.checks, "multi")222        names = try await ArchiveBrowser.list(multi.url, format: .tgz).entries.map(\.path)223        #expect(names.contains("loose.txt") && names.contains("tree/hello.txt") && names.contains("tree 2/b.txt"))224        // Compression levels: store is larger than maximum, and every level reads back.225        var sizes: [String: Int] = [:]226        for level in ["store", "fast", "normal", "maximum"] {227            for format in [FormatID.zip, .sevenZip] {228                let a = try await runEngine([tree], .folder, format, operation: Self.create, options: ["compression.level": .string(level)], in: scratch)229                expectAllPass(a.checks, "\(format) \(level)")230                sizes["\(format)-\(level)"] = try a.url.resourceValues(forKeys: [.fileSizeKey]).fileSize231            }232        }233        #expect(try #require(sizes["zip-store"]) > 1 << 20 && (sizes["zip-maximum"] ?? 0) < (sizes["zip-fast"] ?? 0) && (sizes["7z-maximum"] ?? .max) < (sizes["7z-store"] ?? 0) / 4)234        await #expect(throws: ConversionError.self) { _ = try await runEngine([tree], .folder, .tgz, operation: Self.create, options: ["compression.level": .string("store")], in: scratch) }235        // Nothing to archive is an error, not an empty archive.236        let empty = try scratch.directory("empty/only-noise")237        try Data("x".utf8).write(to: empty.appendingPathComponent(".DS_Store"))238        await #expect(throws: ConversionError.self) { _ = try await runEngine([empty], .folder, .zip, operation: Self.create, options: ["root.keep": .bool(false)], in: scratch) }239    }240241    @Test func extendedAttributesTravelInTarWhenAsked() async throws {242        let scratch = try Scratch(); defer { scratch.cleanUp() }243        let tree = try Tree.make(at: scratch.url("tree"), bigMegabytes: 1)244        let file = tree.appendingPathComponent("hello.txt").path245        _ = Data("blue".utf8).withUnsafeBytes { setxattr(file, "ai.spboucher.colour", $0.baseAddress, 4, 0, 0) }246        let with = try await runEngine([tree], .folder, .txz, operation: Self.create, options: ["xattrs": .bool(true)], in: scratch)247        let restored = try await runEngine([with.url], .txz, .folder, operation: Self.extract, options: ["xattrs": .bool(true)], name: "r", in: scratch)248        #expect(SecureExtractor.readXattr(restored.url.appendingPathComponent("tree/hello.txt").path, "ai.spboucher.colour") == Data("blue".utf8))249        let ignored = try await runEngine([with.url], .txz, .folder, operation: Self.extract, name: "i", in: scratch)250        #expect(SecureExtractor.readXattr(ignored.url.appendingPathComponent("tree/hello.txt").path, "ai.spboucher.colour") == nil)251        let without = try await runEngine([tree], .folder, .txz, operation: Self.create, in: scratch)252        let plain = try await runEngine([without.url], .txz, .folder, operation: Self.extract, options: ["xattrs": .bool(true)], name: "p", in: scratch)253        #expect(SecureExtractor.readXattr(plain.url.appendingPathComponent("tree/hello.txt").path, "ai.spboucher.colour") == nil)254        // ZIP cannot hold them: said before converting.255        let request = ConversionRequest(source: tree, sourceFormat: .folder, destinationFormat: .zip, operation: Self.create, options: ConversionOptions(["xattrs": .bool(true)]))256        #expect(ArchiveEngine().preflight(request, inspection: nil).contains { $0.code == "xattrs-unsupported" })257    }258259    @Test(arguments: ["aes256", "zipcrypto"])260    func encryptedZipRoundTripsAndRefusesWrongPasswords(method: String) async throws {261        let scratch = try Scratch(); defer { scratch.cleanUp() }262        let tree = try Tree.make(at: scratch.url("tree"), bigMegabytes: 1)263        let password = "correct horse — バッテリー"264        let archive = try await runEngine([tree], .folder, .zip, operation: Self.create, options: ["password": .string(password), "encrypt": .bool(true), "encryption": .string(method)], in: scratch)265        expectAllPass(archive.checks, "encrypted \(method)")266        #expect(archive.checks.contains { $0.name == "Every file is encrypted" && $0.passed })267        // The content is really unreadable without the password: the log lines are nowhere in the file.268        let raw = try Data(contentsOf: archive.url)269        #expect(raw.range(of: Data("Hello, archive!".utf8)) == nil && raw.range(of: Data("INFO request".utf8)) == nil)270        let page = try await ArchiveBrowser.list(archive.url, format: .zip)271        #expect(page.summary.hasEncryptedEntries && page.entries.contains { $0.isEncrypted && $0.path == "tree/hello.txt" })272        let inspection = try await ArchiveEngine().inspect(archive.url, format: .zip)273        #expect(inspection.isEncrypted == true)274275        let good = try await runEngine([archive.url], .zip, .folder, operation: Self.extract, options: ["password": .string(password)], name: "good", in: scratch)276        expectAllPass(good.checks, "decrypt \(method)")277        try Tree.expectEqual(tree, good.url.appendingPathComponent("tree"), "decrypted \(method)")278        for (wrong, expected) in [("nope", "not the right one"), ("", "protected by a password")] {279            let work = try scratch.work()280            let request = ConversionRequest(source: archive.url, sourceFormat: .zip, destinationFormat: .folder, operation: Self.extract, options: ConversionOptions(["password": .string(wrong)]), outputBaseName: "bad")281            do {282                _ = try await ArchiveEngine().convert(request, context: .init(workDirectory: work))283                Issue.record("extracted with password “\(wrong)”")284            } catch ConversionError.encryptedInput(let message) {285                #expect(message.contains(expected), "\(message)")286                #expect(!message.contains(password))287            }288            #expect(try FileManager.default.contentsOfDirectory(atPath: work.path).isEmpty, "partial output left behind")289            let warnings = ArchiveEngine().preflight(request, inspection: inspection)290            #expect(warnings.contains { $0.code == "password-needed" } == wrong.isEmpty)291        }292        // An independent implementation opens an archive made the same way. (ASCII password: the test harness hands293        // arguments to the tool in decomposed Unicode, which would not be the bytes the engine used.)294        let ascii = try await runEngine([tree], .folder, .zip, operation: Self.create, options: ["password": .string("Tr0ub4dor&3"), "encrypt": .bool(true), "encryption": .string(method)], in: scratch)295        let system = try Fixture.systemExtract(ascii.url, .zip, in: scratch, password: "Tr0ub4dor&3")296        try Tree.expectEqual(tree, system.appendingPathComponent("tree"), directoryTimes: false, "engine → bsdtar, \(method)")297        // Conversion to another format decrypts with the password; the result is not encrypted.298        let converted = try await runEngine([archive.url], .zip, .tzst, operation: nil, options: ["password": .string(password)], in: scratch)299        expectAllPass(converted.checks, "encrypted zip → tzst")300        #expect(try await ArchiveBrowser.list(converted.url, format: .tzst).summary.hasEncryptedEntries == false)301        let verified = try await ArchiveVerifier.verify(archive.url, format: .zip, password: password)302        #expect(verified.passed && verified.checkedEntries == 8)303        await #expect(throws: ConversionError.self) { _ = try await ArchiveVerifier.verify(archive.url, format: .zip, password: "wrong") }304    }305306    @Test func zipCryptoArchiveMadeByTheSystemZipCommandOpensWithItsPassword() async throws {307        let scratch = try Scratch(); defer { scratch.cleanUp() }308        let tree = try Tree.make(at: scratch.url("tree"), bigMegabytes: 1)309        let zip = scratch.url("system-encrypted.zip")310        try Tool.run("/usr/bin/zip", ["-r", "-y", "-q", "-P", "s3cret", zip.path, "tree"], cwd: tree.deletingLastPathComponent())311        let out = try await runEngine([zip], .zip, .folder, operation: Self.extract, options: ["password": .string("s3cret")], name: "out", in: scratch)312        expectAllPass(out.checks, "system zipcrypto")313        // Info-ZIP encrypts the targets of symbolic links too; the decoder cannot decipher those, so the two links314        // of the tree are left out — and reported — rather than created with garbage targets.315        #expect(out.collector.message("symlinks-encrypted").hasPrefix("2 symbolic links have an encrypted target"))316        try FileManager.default.removeItem(at: tree.appendingPathComponent("link-to-hello"))317        try FileManager.default.removeItem(at: tree.appendingPathComponent("dir with spaces/up-link"))318        try Tree.expectEqual(tree, out.url.appendingPathComponent("tree"), directoryTimes: false, "zip -P → engine")319        #expect(try await ArchiveBrowser.list(zip, format: .zip).entries.filter { $0.kind == .symlink }.allSatisfy { $0.linkTarget == nil && $0.isEncrypted })320        await #expect(throws: ConversionError.self) { _ = try await runEngine([zip], .zip, .folder, operation: Self.extract, options: ["password": .string("wrong")], name: "bad", in: scratch) }321    }322323    @Test func encryptionOptionsAreChecked() async throws {324        let scratch = try Scratch(); defer { scratch.cleanUp() }325        let tree = try Tree.make(at: scratch.url("tree"), bigMegabytes: 1)326        func attempt(_ to: FormatID, _ options: [String: OptionValue]) async throws { _ = try await runEngine([tree], .folder, to, operation: Self.create, options: options, in: scratch) }327        await #expect(throws: ConversionError.invalidOption("Encryption needs a password.")) { try await attempt(.zip, ["encrypt": .bool(true)]) }328        await #expect(throws: ConversionError.invalidOption("Only ZIP archives can be encrypted by this app.")) { try await attempt(.sevenZip, ["encrypt": .bool(true), "password": .string("x")]) }329        await #expect(throws: ConversionError.self) { try await attempt(.zip, ["encrypt": .bool(true), "password": .string("x"), "encryption": .string("rot13")]) }330        await #expect(throws: ConversionError.self) { try await attempt(.zip, ["compression.level": .string("ultra")]) }331        await #expect(throws: ConversionError.self) { try await attempt(.zip, ["limit.ratio": .int(-1)]) }332        // A password without "encrypt" does not encrypt (it is only for reading).333        let plain = try await runEngine([tree], .folder, .zip, operation: Self.create, options: ["password": .string("unused")], in: scratch)334        #expect(try await ArchiveBrowser.list(plain.url, format: .zip).summary.hasEncryptedEntries == false)335        let request = ConversionRequest(source: tree, sourceFormat: .folder, destinationFormat: .zip, operation: Self.create,336                                        options: ConversionOptions(["encrypt": .bool(true), "password": .string("pw"), "encryption": .string("zipcrypto")]))337        let codes = Set(ArchiveEngine().preflight(request, inspection: nil).map(\.code))338        #expect(codes == ["encryption-weak", "encryption-names-visible"])339    }340}341
Files main
Menu

Theme