SPB Git

Repository Public License

FolderToolsTests.swift 16.3 KB 321 lines swift

1// ORIX Secure2// File: Packages/ORIXFileInspection/Tests/ORIXFileInspectionTests/FolderToolsTests.swift3// Description: Duplicate finder, large-file scanner and file remover on folders built for each test.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import ORIXSecureModel9import Testing10@testable import ORIXFileInspection1112@Suite("Duplicates, large files, removal")13struct FolderToolsTests {14    final class Log<Value: Sendable>: @unchecked Sendable {15        private let lock = NSLock()16        private var values: [Value] = []17        func add(_ value: Value) { lock.lock(); values.append(value); lock.unlock() }18        var all: [Value] { lock.lock(); defer { lock.unlock() }; return values }19    }2021    func pattern(_ count: Int, seed: UInt64) -> [UInt8] {22        var generator = SplitMix64(seed: seed)23        return (0..<count).map { _ in UInt8.random(in: 0...255, using: &generator) }24    }2526    func names(_ group: DuplicateGroup) -> [String] { group.urls.map(\.lastPathComponent).sorted() }2728    // MARK: Duplicates2930    @Test("Duplicates across nested folders, with look-alikes that are not duplicates")31    func duplicates() async throws {32        let dir = try TempDir()33        let small = pattern(5000, seed: 1)34        let large = pattern(200_000, seed: 2)35        var largeTail = large36        largeTail[199_999] ^= 0xFF37        var largeMiddle = large38        largeMiddle[70_000] ^= 0x0139        try dir.write("a/small-1.bin", small)40        try dir.write("a/b/c/small-2.bin", small)41        try dir.write("d/small-3.bin", small)42        try dir.write("a/same-size-other-content.bin", pattern(5000, seed: 3))43        try dir.write("large-1.bin", large)44        try dir.write("x/y/large-2.bin", large)45        try dir.write("x/large-differs-at-end.bin", largeTail)46        try dir.write("x/large-differs-after-prefix.bin", largeMiddle)47        try dir.write("unique.txt", bytes("only one of these"))48        try dir.write("empty-1", [])49        try dir.write("z/empty-2", [])50        try dir.write(".hidden-copy.bin", small)5152        let log = Log<DuplicateScanProgress.Phase>()53        let report = try await DuplicateFinder().findDuplicates(inFolder: dir.url) { log.add($0.phase) }54        #expect(report.filesExamined == 11)55        #expect(report.groups.count == 2)56        let first = try #require(report.groups.first)57        #expect(names(first) == ["large-1.bin", "large-2.bin"])58        #expect(first.fileSize == 200_000)59        #expect(first.reclaimableBytes == 200_000)60        #expect(first.sha256 == FileHasher.sha256(of: Data(large)))61        let second = report.groups[1]62        #expect(names(second) == ["small-1.bin", "small-2.bin", "small-3.bin"])63        #expect(second.reclaimableBytes == 10_000)64        #expect(second.sha256 == FileHasher.sha256(of: Data(small)))65        #expect(report.reclaimableBytes == 210_000)66        #expect(report.duplicateFileCount == 3)67        #expect(!report.truncated)68        #expect(report.skipped.isEmpty)69        // Both look-alikes share the first 64 KiB with the real copies, so all four large files are hashed in full.70        let expectedBytes: UInt64 = 20_000 + 262_144 + 800_00071        #expect(report.bytesHashed == expectedBytes)72        #expect(log.all.last == .done)73        #expect(log.all.contains(.hashingPrefixes))74        for text in report.notInspected { for word in forbiddenWords { #expect(!text.lowercased().contains(word)) } }75    }7677    @Test("Empty files and hidden files follow the options")78    func emptyAndHidden() async throws {79        let dir = try TempDir()80        let content = pattern(3000, seed: 9)81        try dir.write("one.bin", content)82        try dir.write(".two.bin", content)83        try dir.write("empty-1", [])84        try dir.write("sub/empty-2", [])85        try dir.write("sub/empty-3", [])86        let standard = try await DuplicateFinder().findDuplicates(inFolder: dir.url)87        #expect(standard.groups.isEmpty)88        var options = DuplicateScanOptions()89        options.minimumFileSize = 090        options.skipHiddenFiles = false91        let everything = try await DuplicateFinder().findDuplicates(inFolder: dir.url, options: options)92        #expect(everything.groups.count == 2)93        let empties = try #require(everything.groups.first { $0.fileSize == 0 })94        #expect(empties.urls.count == 3)95        #expect(empties.reclaimableBytes == 0)96        #expect(empties.sha256 == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")97        #expect(everything.reclaimableBytes == 3000)98    }99100    @Test("An explicit list: missing files, folders, repeats and hard links")101    func explicitList() async throws {102        let dir = try TempDir()103        let content = pattern(80_000, seed: 5)104        let a = try dir.write("a.bin", content)105        let b = try dir.write("b.bin", content)106        let c = try dir.write("c.bin", pattern(80_000, seed: 6))107        let link = dir.url.appendingPathComponent("a-hardlink.bin")108        try FileManager.default.linkItem(at: a, to: link)109        let symlink = dir.url.appendingPathComponent("a-symlink.bin")110        try FileManager.default.createSymbolicLink(at: symlink, withDestinationURL: a)111        let missing = dir.url.appendingPathComponent("missing.bin")112        let report = try await DuplicateFinder().findDuplicates(among: [a, b, c, a, link, symlink, missing, dir.url])113        #expect(report.filesExamined == 4)114        #expect(report.groups.count == 1)115        #expect(report.groups.first?.urls.count == 2)116        #expect(report.hardLinksIgnored == 1)117        #expect(report.reclaimableBytes == 80_000)118        #expect(Set(report.skipped.map(\.url.lastPathComponent)) == ["a-symlink.bin", "missing.bin", dir.url.lastPathComponent])119        let none = try await DuplicateFinder().findDuplicates(among: [])120        #expect(none.groups.isEmpty)121    }122123    @Test("The file limit, packages, a non-folder and cancellation")124    func limitsAndCancellation() async throws {125        let dir = try TempDir()126        for index in 0..<40 { try dir.write("f\(index).bin", pattern(2000, seed: UInt64(index % 4))) }127        try dir.write("Bundle.app/Contents/copy.bin", pattern(2000, seed: 0))128        var options = DuplicateScanOptions()129        options.maxFiles = 10130        let limited = try await DuplicateFinder().findDuplicates(inFolder: dir.url, options: options)131        #expect(limited.truncated)132        #expect(limited.filesExamined == 10)133        #expect(limited.notInspected.contains { $0.contains("stopped after") })134        let full = try await DuplicateFinder().findDuplicates(inFolder: dir.url)135        #expect(full.filesExamined == 40)136        #expect(full.groups.count == 4)137        #expect(full.reclaimableBytes == 36 * 2000)138        options = DuplicateScanOptions()139        options.skipPackages = false140        let inside = try await DuplicateFinder().findDuplicates(inFolder: dir.url, options: options)141        #expect(inside.filesExamined == 41)142143        let file = try dir.write("plain.txt", bytes("x"))144        await #expect(throws: FolderScanError.self) { _ = try await DuplicateFinder().findDuplicates(inFolder: file) }145146        let task = Task { try await DuplicateFinder().findDuplicates(inFolder: dir.url) { _ in withUnsafeCurrentTask { $0?.cancel() } } }147        let result = await task.result148        #expect(throws: CancellationError.self) { try result.get() }149    }150151    // MARK: Large files152153    @Test("Largest files, per-extension totals and limits")154    func largeFiles() async throws {155        let dir = try TempDir()156        let sizes: [(String, Int)] = [("video.mov", 900_000), ("a/photo1.jpg", 300_000), ("a/photo2.JPG", 200_000), ("a/b/doc.pdf", 50_000), ("notes.txt", 1000), ("README", 10), (".cache.tmp", 600_000)]157        for (name, size) in sizes { try dir.write(name, [UInt8](repeating: 0xAB, count: size)) }158        try dir.write("empty.txt", [])159        var options = LargeFileScanOptions()160        options.resultCount = 3161        let report = try await LargeFileScanner().scan(folder: dir.url, options: options)162        #expect(report.filesExamined == 8)163        #expect(report.largestFiles.map(\.url.lastPathComponent) == ["video.mov", ".cache.tmp", "photo1.jpg"])164        #expect(report.largestFiles.map(\.logicalSize) == [900_000, 600_000, 300_000])165        #expect(report.totalLogicalBytes == 2_051_010)166        #expect(report.totalBytes >= report.totalLogicalBytes)167        #expect(report.filesWithoutAllocatedSize == 0)168        let first = try #require(report.largestFiles.first)169        #expect((first.allocatedSize ?? 0) >= 900_000)170        #expect(first.size == first.allocatedSize)171        #expect(report.extensionTotals.map(\.fileExtension).prefix(3) == ["mov", "tmp", "jpg"])172        let jpg = try #require(report.extensionTotals.first { $0.fileExtension == "jpg" })173        #expect(jpg.fileCount == 2)174        let none = try #require(report.extensionTotals.first { $0.fileExtension == "" })175        #expect(none.fileCount == 1)176        #expect(!report.truncated)177178        options.skipHiddenFiles = true179        options.resultCount = 100180        options.minimumListedSize = 100_000181        let visible = try await LargeFileScanner().scan(folder: dir.url, options: options)182        #expect(visible.filesExamined == 7)183        #expect(visible.largestFiles.count == 3)184        options = LargeFileScanOptions()185        options.maxFiles = 2186        let cut = try await LargeFileScanner().scan(folder: dir.url, options: options)187        #expect(cut.truncated)188        #expect(cut.filesExamined == 2)189        options = LargeFileScanOptions()190        options.resultCount = 0191        let totalsOnly = try await LargeFileScanner().scan(folder: dir.url, options: options)192        #expect(totalsOnly.largestFiles.isEmpty)193        #expect(totalsOnly.totalLogicalBytes == 2_051_010)194        await #expect(throws: FolderScanError.self) { _ = try await LargeFileScanner().scan(folder: dir.url.appendingPathComponent("absent")) }195    }196197    @Test("A folder of 5 000 files is scanned and ranked")198    func largeFolderTiming() async throws {199        let dir = try TempDir()200        var generator = SplitMix64(seed: 11)201        var expected: [Int] = []202        var allocated: [Int64] = []203        for index in 0..<5000 {204            let size = Int.random(in: 0...60_000, using: &generator)205            expected.append(size)206            let url = try dir.write("d\(index % 50)/s\(index % 7)/file\(index).\(["txt", "dat", "log"][index % 3])", [UInt8](repeating: 1, count: size))207            let values = try url.resourceValues(forKeys: [.totalFileAllocatedSizeKey])208            allocated.append(Int64(values.totalFileAllocatedSize ?? size))209        }210        let scanner = LargeFileScanner()211        let report = try await scanner.scan(folder: dir.url)212        print("Large-file scan: 5 000 files in \(String(format: "%.3f", report.duration)) s (\(String(format: "%.0f", 5000 / report.duration)) files/s)")213        #expect(report.filesExamined == 5000)214        #expect(report.largestFiles.count == 50)215        let ranked = report.largestFiles.map { $0.size }216        #expect(ranked == Array(allocated.sorted(by: >).prefix(50)))217        #expect(report.totalLogicalBytes == Int64(expected.reduce(0, +)))218        #expect(report.extensionTotals.count == 3)219        #expect(report.extensionTotals.reduce(0) { $0 + $1.fileCount } == 5000)220        let start = Date()221        let duplicates = try await DuplicateFinder().findDuplicates(inFolder: dir.url)222        print("Duplicate scan: 5 000 files in \(String(format: "%.3f", Date().timeIntervalSince(start))) s, \(duplicates.groups.count) groups")223        #expect(duplicates.filesExamined == 5000)224        #expect(duplicates.groups.allSatisfy { $0.urls.count > 1 })225    }226227    // MARK: Removal228229    @Test("Only the listed files are removed, and freed bytes are verified")230    func removal() async throws {231        let dir = try TempDir()232        let a = try dir.write("a.bin", [UInt8](repeating: 1, count: 100_000))233        let b = try dir.write("sub/b.bin", [UInt8](repeating: 2, count: 20_000))234        let keep = try dir.write("keep.bin", [UInt8](repeating: 3, count: 5000))235        let sibling = try dir.write("sub/sibling.bin", [UInt8](repeating: 4, count: 5000))236        let missing = dir.url.appendingPathComponent("missing.bin")237        let folder = dir.url.appendingPathComponent("sub")238        let report = await FileRemover().remove([a, missing, b, folder, a])239        #expect(report.results.count == 4)240        #expect(report.results.map(\.outcome) == [.removed, .missing, .removed, .refusedDirectory])241        #expect(report.removedCount == 2)242        #expect(report.failedCount == 2)243        #expect(report.results[0].bytesFreed >= 100_000)244        #expect(report.results[1].bytesFreed == 0)245        #expect(report.results[1].message != nil)246        #expect(report.results[3].bytesFreed == 0)247        #expect(report.bytesFreed == report.results[0].bytesFreed + report.results[2].bytesFreed)248        #expect(!report.cancelled)249        let fm = FileManager.default250        #expect(!fm.fileExists(atPath: a.path))251        #expect(!fm.fileExists(atPath: b.path))252        #expect(fm.fileExists(atPath: keep.path))253        #expect(fm.fileExists(atPath: sibling.path))254        #expect(fm.fileExists(atPath: folder.path))255        let nothing = await FileRemover().remove([])256        #expect(nothing.results.isEmpty)257        #expect(nothing.bytesFreed == 0)258    }259260    @Test("A file in a read-only folder is reported as failed and stays in place")261    func readOnlyParent() async throws {262        let dir = try TempDir()263        let locked = try dir.write("locked/file.bin", [UInt8](repeating: 9, count: 30_000))264        let parent = locked.deletingLastPathComponent()265        #expect(chmod(parent.path, 0o555) == 0)266        defer { _ = chmod(parent.path, 0o755) }267        let free = try dir.write("free.bin", [UInt8](repeating: 9, count: 30_000))268        let report = await FileRemover().remove([locked, free])269        #expect(report.results[0].outcome == .failed)270        #expect(report.results[0].bytesFreed == 0)271        #expect(report.results[0].message?.isEmpty == false)272        #expect(report.results[1].outcome == .removed)273        #expect(FileManager.default.fileExists(atPath: locked.path))274        #expect(report.bytesFreed == report.results[1].bytesFreed)275    }276277    @Test("Folders need the explicit flag; links are removed, never their target")278    func foldersAndLinks() async throws {279        let dir = try TempDir()280        let target = try dir.write("target.bin", [UInt8](repeating: 5, count: 40_000))281        let symlink = dir.url.appendingPathComponent("link.bin")282        try FileManager.default.createSymbolicLink(at: symlink, withDestinationURL: target)283        let hardlink = dir.url.appendingPathComponent("second-name.bin")284        try FileManager.default.linkItem(at: target, to: hardlink)285        try dir.write("tree/one.bin", [UInt8](repeating: 6, count: 10_000))286        try dir.write("tree/deep/two.bin", [UInt8](repeating: 7, count: 10_000))287        let tree = dir.url.appendingPathComponent("tree")288289        let links = await FileRemover().remove([symlink, hardlink])290        #expect(links.results.map(\.outcome) == [.removed, .removed])291        #expect(links.results[1].bytesFreed == 0)292        #expect(links.results[1].message != nil)293        #expect(FileManager.default.fileExists(atPath: target.path))294        let data = try Data(contentsOf: target)295        #expect(data.count == 40_000)296297        let refused = await FileRemover().remove([tree])298        #expect(refused.results.first?.outcome == .refusedDirectory)299        #expect(FileManager.default.fileExists(atPath: tree.appendingPathComponent("deep/two.bin").path))300        let allowed = await FileRemover().remove([tree], allowDirectories: true)301        #expect(allowed.results.first?.outcome == .removed)302        #expect((allowed.results.first?.bytesFreed ?? 0) >= 20_000)303        #expect(!FileManager.default.fileExists(atPath: tree.path))304        #expect(FileManager.default.fileExists(atPath: target.path))305    }306307    @Test("A cancelled removal touches nothing further")308    func cancelledRemoval() async throws {309        let dir = try TempDir()310        let files = try (0..<5).map { try dir.write("f\($0).bin", [UInt8](repeating: 1, count: 100)) }311        let task = Task { () -> RemovalReport in312            withUnsafeCurrentTask { $0?.cancel() }313            return await FileRemover().remove(files)314        }315        let report = await task.value316        #expect(report.cancelled)317        #expect(report.results.isEmpty)318        for file in files { #expect(FileManager.default.fileExists(atPath: file.path)) }319    }320}321
Files main
Menu

Theme