Repository Public License
1// ORIX Secure2// File: Packages/ORIXFileInspection/Tests/ORIXFileInspectionTests/FolderToolsTests.swift3// Description: Duplicate finder, large-file scanner and file remover on folders built for each test.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import ORIXSecureModel9import Testing10@testable import ORIXFileInspection1112@Suite("Duplicates, large files, removal")13struct FolderToolsTests {14 final class Log<Value: Sendable>: @unchecked Sendable {15 private let lock = NSLock()16 private var values: [Value] = []17 func add(_ value: Value) { lock.lock(); values.append(value); lock.unlock() }18 var all: [Value] { lock.lock(); defer { lock.unlock() }; return values }19 }2021 func pattern(_ count: Int, seed: UInt64) -> [UInt8] {22 var generator = SplitMix64(seed: seed)23 return (0..<count).map { _ in UInt8.random(in: 0...255, using: &generator) }24 }2526 func names(_ group: DuplicateGroup) -> [String] { group.urls.map(\.lastPathComponent).sorted() }2728 // MARK: Duplicates2930 @Test("Duplicates across nested folders, with look-alikes that are not duplicates")31 func duplicates() async throws {32 let dir = try TempDir()33 let small = pattern(5000, seed: 1)34 let large = pattern(200_000, seed: 2)35 var largeTail = large36 largeTail[199_999] ^= 0xFF37 var largeMiddle = large38 largeMiddle[70_000] ^= 0x0139 try dir.write("a/small-1.bin", small)40 try dir.write("a/b/c/small-2.bin", small)41 try dir.write("d/small-3.bin", small)42 try dir.write("a/same-size-other-content.bin", pattern(5000, seed: 3))43 try dir.write("large-1.bin", large)44 try dir.write("x/y/large-2.bin", large)45 try dir.write("x/large-differs-at-end.bin", largeTail)46 try dir.write("x/large-differs-after-prefix.bin", largeMiddle)47 try dir.write("unique.txt", bytes("only one of these"))48 try dir.write("empty-1", [])49 try dir.write("z/empty-2", [])50 try dir.write(".hidden-copy.bin", small)5152 let log = Log<DuplicateScanProgress.Phase>()53 let report = try await DuplicateFinder().findDuplicates(inFolder: dir.url) { log.add($0.phase) }54 #expect(report.filesExamined == 11)55 #expect(report.groups.count == 2)56 let first = try #require(report.groups.first)57 #expect(names(first) == ["large-1.bin", "large-2.bin"])58 #expect(first.fileSize == 200_000)59 #expect(first.reclaimableBytes == 200_000)60 #expect(first.sha256 == FileHasher.sha256(of: Data(large)))61 let second = report.groups[1]62 #expect(names(second) == ["small-1.bin", "small-2.bin", "small-3.bin"])63 #expect(second.reclaimableBytes == 10_000)64 #expect(second.sha256 == FileHasher.sha256(of: Data(small)))65 #expect(report.reclaimableBytes == 210_000)66 #expect(report.duplicateFileCount == 3)67 #expect(!report.truncated)68 #expect(report.skipped.isEmpty)69 // Both look-alikes share the first 64 KiB with the real copies, so all four large files are hashed in full.70 let expectedBytes: UInt64 = 20_000 + 262_144 + 800_00071 #expect(report.bytesHashed == expectedBytes)72 #expect(log.all.last == .done)73 #expect(log.all.contains(.hashingPrefixes))74 for text in report.notInspected { for word in forbiddenWords { #expect(!text.lowercased().contains(word)) } }75 }7677 @Test("Empty files and hidden files follow the options")78 func emptyAndHidden() async throws {79 let dir = try TempDir()80 let content = pattern(3000, seed: 9)81 try dir.write("one.bin", content)82 try dir.write(".two.bin", content)83 try dir.write("empty-1", [])84 try dir.write("sub/empty-2", [])85 try dir.write("sub/empty-3", [])86 let standard = try await DuplicateFinder().findDuplicates(inFolder: dir.url)87 #expect(standard.groups.isEmpty)88 var options = DuplicateScanOptions()89 options.minimumFileSize = 090 options.skipHiddenFiles = false91 let everything = try await DuplicateFinder().findDuplicates(inFolder: dir.url, options: options)92 #expect(everything.groups.count == 2)93 let empties = try #require(everything.groups.first { $0.fileSize == 0 })94 #expect(empties.urls.count == 3)95 #expect(empties.reclaimableBytes == 0)96 #expect(empties.sha256 == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")97 #expect(everything.reclaimableBytes == 3000)98 }99100 @Test("An explicit list: missing files, folders, repeats and hard links")101 func explicitList() async throws {102 let dir = try TempDir()103 let content = pattern(80_000, seed: 5)104 let a = try dir.write("a.bin", content)105 let b = try dir.write("b.bin", content)106 let c = try dir.write("c.bin", pattern(80_000, seed: 6))107 let link = dir.url.appendingPathComponent("a-hardlink.bin")108 try FileManager.default.linkItem(at: a, to: link)109 let symlink = dir.url.appendingPathComponent("a-symlink.bin")110 try FileManager.default.createSymbolicLink(at: symlink, withDestinationURL: a)111 let missing = dir.url.appendingPathComponent("missing.bin")112 let report = try await DuplicateFinder().findDuplicates(among: [a, b, c, a, link, symlink, missing, dir.url])113 #expect(report.filesExamined == 4)114 #expect(report.groups.count == 1)115 #expect(report.groups.first?.urls.count == 2)116 #expect(report.hardLinksIgnored == 1)117 #expect(report.reclaimableBytes == 80_000)118 #expect(Set(report.skipped.map(\.url.lastPathComponent)) == ["a-symlink.bin", "missing.bin", dir.url.lastPathComponent])119 let none = try await DuplicateFinder().findDuplicates(among: [])120 #expect(none.groups.isEmpty)121 }122123 @Test("The file limit, packages, a non-folder and cancellation")124 func limitsAndCancellation() async throws {125 let dir = try TempDir()126 for index in 0..<40 { try dir.write("f\(index).bin", pattern(2000, seed: UInt64(index % 4))) }127 try dir.write("Bundle.app/Contents/copy.bin", pattern(2000, seed: 0))128 var options = DuplicateScanOptions()129 options.maxFiles = 10130 let limited = try await DuplicateFinder().findDuplicates(inFolder: dir.url, options: options)131 #expect(limited.truncated)132 #expect(limited.filesExamined == 10)133 #expect(limited.notInspected.contains { $0.contains("stopped after") })134 let full = try await DuplicateFinder().findDuplicates(inFolder: dir.url)135 #expect(full.filesExamined == 40)136 #expect(full.groups.count == 4)137 #expect(full.reclaimableBytes == 36 * 2000)138 options = DuplicateScanOptions()139 options.skipPackages = false140 let inside = try await DuplicateFinder().findDuplicates(inFolder: dir.url, options: options)141 #expect(inside.filesExamined == 41)142143 let file = try dir.write("plain.txt", bytes("x"))144 await #expect(throws: FolderScanError.self) { _ = try await DuplicateFinder().findDuplicates(inFolder: file) }145146 let task = Task { try await DuplicateFinder().findDuplicates(inFolder: dir.url) { _ in withUnsafeCurrentTask { $0?.cancel() } } }147 let result = await task.result148 #expect(throws: CancellationError.self) { try result.get() }149 }150151 // MARK: Large files152153 @Test("Largest files, per-extension totals and limits")154 func largeFiles() async throws {155 let dir = try TempDir()156 let sizes: [(String, Int)] = [("video.mov", 900_000), ("a/photo1.jpg", 300_000), ("a/photo2.JPG", 200_000), ("a/b/doc.pdf", 50_000), ("notes.txt", 1000), ("README", 10), (".cache.tmp", 600_000)]157 for (name, size) in sizes { try dir.write(name, [UInt8](repeating: 0xAB, count: size)) }158 try dir.write("empty.txt", [])159 var options = LargeFileScanOptions()160 options.resultCount = 3161 let report = try await LargeFileScanner().scan(folder: dir.url, options: options)162 #expect(report.filesExamined == 8)163 #expect(report.largestFiles.map(\.url.lastPathComponent) == ["video.mov", ".cache.tmp", "photo1.jpg"])164 #expect(report.largestFiles.map(\.logicalSize) == [900_000, 600_000, 300_000])165 #expect(report.totalLogicalBytes == 2_051_010)166 #expect(report.totalBytes >= report.totalLogicalBytes)167 #expect(report.filesWithoutAllocatedSize == 0)168 let first = try #require(report.largestFiles.first)169 #expect((first.allocatedSize ?? 0) >= 900_000)170 #expect(first.size == first.allocatedSize)171 #expect(report.extensionTotals.map(\.fileExtension).prefix(3) == ["mov", "tmp", "jpg"])172 let jpg = try #require(report.extensionTotals.first { $0.fileExtension == "jpg" })173 #expect(jpg.fileCount == 2)174 let none = try #require(report.extensionTotals.first { $0.fileExtension == "" })175 #expect(none.fileCount == 1)176 #expect(!report.truncated)177178 options.skipHiddenFiles = true179 options.resultCount = 100180 options.minimumListedSize = 100_000181 let visible = try await LargeFileScanner().scan(folder: dir.url, options: options)182 #expect(visible.filesExamined == 7)183 #expect(visible.largestFiles.count == 3)184 options = LargeFileScanOptions()185 options.maxFiles = 2186 let cut = try await LargeFileScanner().scan(folder: dir.url, options: options)187 #expect(cut.truncated)188 #expect(cut.filesExamined == 2)189 options = LargeFileScanOptions()190 options.resultCount = 0191 let totalsOnly = try await LargeFileScanner().scan(folder: dir.url, options: options)192 #expect(totalsOnly.largestFiles.isEmpty)193 #expect(totalsOnly.totalLogicalBytes == 2_051_010)194 await #expect(throws: FolderScanError.self) { _ = try await LargeFileScanner().scan(folder: dir.url.appendingPathComponent("absent")) }195 }196197 @Test("A folder of 5 000 files is scanned and ranked")198 func largeFolderTiming() async throws {199 let dir = try TempDir()200 var generator = SplitMix64(seed: 11)201 var expected: [Int] = []202 var allocated: [Int64] = []203 for index in 0..<5000 {204 let size = Int.random(in: 0...60_000, using: &generator)205 expected.append(size)206 let url = try dir.write("d\(index % 50)/s\(index % 7)/file\(index).\(["txt", "dat", "log"][index % 3])", [UInt8](repeating: 1, count: size))207 let values = try url.resourceValues(forKeys: [.totalFileAllocatedSizeKey])208 allocated.append(Int64(values.totalFileAllocatedSize ?? size))209 }210 let scanner = LargeFileScanner()211 let report = try await scanner.scan(folder: dir.url)212 print("Large-file scan: 5 000 files in \(String(format: "%.3f", report.duration)) s (\(String(format: "%.0f", 5000 / report.duration)) files/s)")213 #expect(report.filesExamined == 5000)214 #expect(report.largestFiles.count == 50)215 let ranked = report.largestFiles.map { $0.size }216 #expect(ranked == Array(allocated.sorted(by: >).prefix(50)))217 #expect(report.totalLogicalBytes == Int64(expected.reduce(0, +)))218 #expect(report.extensionTotals.count == 3)219 #expect(report.extensionTotals.reduce(0) { $0 + $1.fileCount } == 5000)220 let start = Date()221 let duplicates = try await DuplicateFinder().findDuplicates(inFolder: dir.url)222 print("Duplicate scan: 5 000 files in \(String(format: "%.3f", Date().timeIntervalSince(start))) s, \(duplicates.groups.count) groups")223 #expect(duplicates.filesExamined == 5000)224 #expect(duplicates.groups.allSatisfy { $0.urls.count > 1 })225 }226227 // MARK: Removal228229 @Test("Only the listed files are removed, and freed bytes are verified")230 func removal() async throws {231 let dir = try TempDir()232 let a = try dir.write("a.bin", [UInt8](repeating: 1, count: 100_000))233 let b = try dir.write("sub/b.bin", [UInt8](repeating: 2, count: 20_000))234 let keep = try dir.write("keep.bin", [UInt8](repeating: 3, count: 5000))235 let sibling = try dir.write("sub/sibling.bin", [UInt8](repeating: 4, count: 5000))236 let missing = dir.url.appendingPathComponent("missing.bin")237 let folder = dir.url.appendingPathComponent("sub")238 let report = await FileRemover().remove([a, missing, b, folder, a])239 #expect(report.results.count == 4)240 #expect(report.results.map(\.outcome) == [.removed, .missing, .removed, .refusedDirectory])241 #expect(report.removedCount == 2)242 #expect(report.failedCount == 2)243 #expect(report.results[0].bytesFreed >= 100_000)244 #expect(report.results[1].bytesFreed == 0)245 #expect(report.results[1].message != nil)246 #expect(report.results[3].bytesFreed == 0)247 #expect(report.bytesFreed == report.results[0].bytesFreed + report.results[2].bytesFreed)248 #expect(!report.cancelled)249 let fm = FileManager.default250 #expect(!fm.fileExists(atPath: a.path))251 #expect(!fm.fileExists(atPath: b.path))252 #expect(fm.fileExists(atPath: keep.path))253 #expect(fm.fileExists(atPath: sibling.path))254 #expect(fm.fileExists(atPath: folder.path))255 let nothing = await FileRemover().remove([])256 #expect(nothing.results.isEmpty)257 #expect(nothing.bytesFreed == 0)258 }259260 @Test("A file in a read-only folder is reported as failed and stays in place")261 func readOnlyParent() async throws {262 let dir = try TempDir()263 let locked = try dir.write("locked/file.bin", [UInt8](repeating: 9, count: 30_000))264 let parent = locked.deletingLastPathComponent()265 #expect(chmod(parent.path, 0o555) == 0)266 defer { _ = chmod(parent.path, 0o755) }267 let free = try dir.write("free.bin", [UInt8](repeating: 9, count: 30_000))268 let report = await FileRemover().remove([locked, free])269 #expect(report.results[0].outcome == .failed)270 #expect(report.results[0].bytesFreed == 0)271 #expect(report.results[0].message?.isEmpty == false)272 #expect(report.results[1].outcome == .removed)273 #expect(FileManager.default.fileExists(atPath: locked.path))274 #expect(report.bytesFreed == report.results[1].bytesFreed)275 }276277 @Test("Folders need the explicit flag; links are removed, never their target")278 func foldersAndLinks() async throws {279 let dir = try TempDir()280 let target = try dir.write("target.bin", [UInt8](repeating: 5, count: 40_000))281 let symlink = dir.url.appendingPathComponent("link.bin")282 try FileManager.default.createSymbolicLink(at: symlink, withDestinationURL: target)283 let hardlink = dir.url.appendingPathComponent("second-name.bin")284 try FileManager.default.linkItem(at: target, to: hardlink)285 try dir.write("tree/one.bin", [UInt8](repeating: 6, count: 10_000))286 try dir.write("tree/deep/two.bin", [UInt8](repeating: 7, count: 10_000))287 let tree = dir.url.appendingPathComponent("tree")288289 let links = await FileRemover().remove([symlink, hardlink])290 #expect(links.results.map(\.outcome) == [.removed, .removed])291 #expect(links.results[1].bytesFreed == 0)292 #expect(links.results[1].message != nil)293 #expect(FileManager.default.fileExists(atPath: target.path))294 let data = try Data(contentsOf: target)295 #expect(data.count == 40_000)296297 let refused = await FileRemover().remove([tree])298 #expect(refused.results.first?.outcome == .refusedDirectory)299 #expect(FileManager.default.fileExists(atPath: tree.appendingPathComponent("deep/two.bin").path))300 let allowed = await FileRemover().remove([tree], allowDirectories: true)301 #expect(allowed.results.first?.outcome == .removed)302 #expect((allowed.results.first?.bytesFreed ?? 0) >= 20_000)303 #expect(!FileManager.default.fileExists(atPath: tree.path))304 #expect(FileManager.default.fileExists(atPath: target.path))305 }306307 @Test("A cancelled removal touches nothing further")308 func cancelledRemoval() async throws {309 let dir = try TempDir()310 let files = try (0..<5).map { try dir.write("f\($0).bin", [UInt8](repeating: 1, count: 100)) }311 let task = Task { () -> RemovalReport in312 withUnsafeCurrentTask { $0?.cancel() }313 return await FileRemover().remove(files)314 }315 let report = await task.value316 #expect(report.cancelled)317 #expect(report.results.isEmpty)318 for file in files { #expect(FileManager.default.fileExists(atPath: file.path)) }319 }320}321