Repository Public License
1// ORIX Clean — Packages/ORIXCleanCore/Tests/ORIXCleanCoreTests/DuplicateAndImageTests.swift2// Description: Duplicate detection and similar-image tests on generated files.3// Author: Simon-Pierre Boucher — contact@spboucher.ai4import CoreGraphics5import Darwin6import Foundation7import ImageIO8import Testing9import UniformTypeIdentifiers10@testable import ORIXCleanCore1112@Suite struct DuplicateTests {13 @Test func exactDuplicatesAreGroupedAndDifferentFilesAreNot() throws {14 let s = Sandbox()15 s.file("a/one.bin", size: 200_000, seed: 1)16 s.file("b/one-copy.bin", size: 200_000, seed: 1)17 s.file("c/one-third.bin", size: 200_000, seed: 1)18 s.file("a/same-size-different.bin", size: 200_000, seed: 2)19 s.file("a/small.bin", size: 10, seed: 3)20 s.file("b/small.bin", size: 10, seed: 3)21 s.file("a/unique.bin", size: 12_345, seed: 4)22 let r = DuplicateFinder().run(tree: try s.scan().tree)23 #expect(r.groups.count == 2)24 #expect(r.groups[0].files.count == 3 && r.groups[0].logicalSize == 200_000)25 #expect(r.groups[0].redundantLogical == 400_000)26 #expect(r.groups[1].files.count == 2 && r.groups[1].logicalSize == 10)27 #expect(r.redundantLogical == 400_010)28 #expect(!r.groups[0].sharesStorage)29 let expected = shell("/usr/bin/shasum", ["-a", "256", s.path("a/one.bin")]).output.split(separator: " ").first.map(String.init)30 #expect(r.groups[0].digest == expected)31 var o = DuplicateOptions(); o.minimumSize = 100032 #expect(DuplicateFinder().run(tree: try s.scan().tree, options: o).groups.count == 1)33 }3435 @Test func differencesOutsideTheSampledWindowsAreCaughtByTheFullHash() throws {36 let s = Sandbox()37 let a = s.file("x.bin", size: 3_000_000, seed: 7)38 let b = s.path("y.bin")39 try FileManager.default.copyItem(atPath: a, toPath: b)40 // Flip one byte far from the head, middle and tail windows. The copy is a clone, so write breaks sharing for one block.41 let fd = open(b, O_WRONLY); var byte: UInt8 = 0xFF42 pwrite(fd, &byte, 1, 700_001); close(fd)43 #expect(DuplicateFinder.sampleHash(a, 3_000_000) == DuplicateFinder.sampleHash(b, 3_000_000))44 #expect(DuplicateFinder().run(tree: try s.scan().tree).groups.isEmpty)45 #expect(!DuplicateFinder.identicalContents(a, b))46 }4748 @Test func collisionsInThePreliminaryFilterDoNotCreateFalseGroups() throws {49 let s = Sandbox()50 for i in 0 ..< 6 { s.file("f\(i).bin", size: 100_000, seed: UInt8(i / 2)) }51 let colliding = DuplicateFinder(sampler: { _, _ in Data([1, 2, 3]) })52 let r = colliding.run(tree: try s.scan().tree)53 #expect(r.groups.count == 3)54 #expect(r.groups.allSatisfy { $0.files.count == 2 })55 }5657 @Test func hardLinksAreOneFileNotDuplicates() throws {58 let s = Sandbox()59 let a = s.file("a.bin", size: 80_000, seed: 9)60 link(a, s.path("a-link.bin"))61 #expect(DuplicateFinder().run(tree: try s.scan().tree).groups.isEmpty)62 s.file("copy.bin", size: 80_000, seed: 9)63 let r = DuplicateFinder().run(tree: try s.scan().tree)64 #expect(r.groups.count == 1)65 #expect(r.groups[0].files.count == 2)66 let linked = r.groups[0].files.first { !$0.hardLinkPaths.isEmpty }!67 #expect(linked.hardLinkPaths.count == 1)68 // Removing one name of a hard-linked file frees nothing.69 #expect(r.groups[0].recoverable(removing: [linked.path]).maximum == 0)70 }7172 @Test func clonesAreDuplicatesThatShareStorage() throws {73 let s = Sandbox()74 let a = s.file("orig.bin", size: 2_000_000, seed: 5)75 clonefile(a, s.path("clone.bin"), 0)76 let r = DuplicateFinder().run(tree: try s.scan().tree)77 #expect(r.groups.count == 1)78 #expect(r.groups[0].sharesStorage)79 let est = r.groups[0].recoverable(removing: [s.path("clone.bin")])80 #expect(est.minimum < 100_000)81 #expect(est.maximum >= 2_000_000)82 #expect(!est.isExact)83 }8485 @Test func largeFilesAndUnreadableFiles() throws {86 let s = Sandbox()87 let big = s.file("big1.bin", size: 48_000_000, seed: 11)88 // A real copy (not a clone) so both are read in full.89 let data = try Data(contentsOf: URL(fileURLWithPath: big))90 FileManager.default.createFile(atPath: s.path("big2.bin"), contents: data)91 s.file("locked1.bin", size: 70_000, seed: 12)92 s.file("locked2.bin", size: 70_000, seed: 12)93 chmod(s.path("locked2.bin"), 0o000)94 let finder = DuplicateFinder()95 let r = finder.run(tree: try s.scan().tree)96 #expect(r.groups.count == 1)97 #expect(r.groups[0].logicalSize == 48_000_000)98 #expect(r.bytesHashed >= 96_000_000)99 #expect(r.unreadable == [s.path("locked2.bin")] || r.unreadable.isEmpty)100 #expect(finder.progress.stage == .done)101 }102103 @Test func aFileThatChangedSinceTheScanIsDropped() throws {104 let s = Sandbox()105 s.file("a.bin", size: 50_000, seed: 1)106 s.file("b.bin", size: 50_000, seed: 1)107 let tree = try s.scan().tree108 let h = FileHandle(forWritingAtPath: s.path("b.bin"))!109 h.seekToEndOfFile(); h.write(Data([1, 2, 3])); try h.close()110 #expect(DuplicateFinder().run(tree: tree).groups.isEmpty)111 #expect(DuplicateFinder.fullHash(path: s.path("b.bin"), expectedSize: 50_000) == nil)112 #expect(DuplicateFinder.fullHash(path: s.path("b.bin")) != nil)113 }114115 @Test func packageContentsAreSkippedUnlessAsked() throws {116 let s = Sandbox()117 s.file("One.app/Contents/Resources/asset.bin", size: 40_000, seed: 3)118 s.file("Two.app/Contents/Resources/asset.bin", size: 40_000, seed: 3)119 let tree = try s.scan().tree120 #expect(DuplicateFinder().run(tree: tree).groups.isEmpty)121 var o = DuplicateOptions(); o.includePackageContents = true122 #expect(DuplicateFinder().run(tree: tree, options: o).groups.count == 1)123 }124125 @Test func cancellationReturnsNoGroups() throws {126 let s = Sandbox()127 for i in 0 ..< 20 { s.file("f\(i).bin", size: 300_000, seed: 1) }128 let f = DuplicateFinder()129 f.cancel()130 let r = f.run(tree: try s.scan().tree)131 #expect(r.cancelled && r.groups.isEmpty)132 }133}134135@Suite struct SimilarImageTests {136 /// A deterministic picture: gradient background and a few shapes that depend on `seed`.137 static func picture(seed: Int, size: Int = 640) -> CGImage {138 let ctx = CGContext(data: nil, width: size, height: size, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpaceCreateDeviceRGB(),139 bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue)!140 var x = UInt64(seed) &* 6364136223846793005 &+ 1442695040888963407141 func rnd() -> CGFloat { x = x &* 6364136223846793005 &+ 1442695040888963407; return CGFloat((x >> 33) % 1000) / 1000 }142 let colors = [CGColor(red: rnd(), green: rnd(), blue: rnd(), alpha: 1), CGColor(red: rnd(), green: rnd(), blue: rnd(), alpha: 1)] as CFArray143 let g = CGGradient(colorsSpace: CGColorSpaceCreateDeviceRGB(), colors: colors, locations: [0, 1])!144 ctx.drawLinearGradient(g, start: .zero, end: CGPoint(x: CGFloat(size) * rnd(), y: CGFloat(size)), options: [.drawsAfterEndLocation])145 for _ in 0 ..< 14 {146 ctx.setFillColor(CGColor(red: rnd(), green: rnd(), blue: rnd(), alpha: 1))147 let r = CGRect(x: rnd() * CGFloat(size), y: rnd() * CGFloat(size), width: rnd() * CGFloat(size) / 2.5 + 20, height: rnd() * CGFloat(size) / 2.5 + 20)148 if rnd() > 0.5 { ctx.fillEllipse(in: r) } else { ctx.fill(r) }149 }150 return ctx.makeImage()!151 }152153 static func write(_ image: CGImage, to path: String, type: UTType = .jpeg, quality: Double = 0.9, scale: CGFloat = 1, crop: CGFloat = 0) {154 var img = image155 if crop > 0 {156 let inset = CGFloat(image.width) * crop157 img = image.cropping(to: CGRect(x: inset, y: inset, width: CGFloat(image.width) - 2 * inset, height: CGFloat(image.height) - 2 * inset))!158 }159 if scale != 1 {160 let w = Int(CGFloat(img.width) * scale), h = Int(CGFloat(img.height) * scale)161 let ctx = CGContext(data: nil, width: w, height: h, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpaceCreateDeviceRGB(),162 bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue)!163 ctx.interpolationQuality = .high164 ctx.draw(img, in: CGRect(x: 0, y: 0, width: w, height: h))165 img = ctx.makeImage()!166 }167 try? FileManager.default.createDirectory(atPath: (path as NSString).deletingLastPathComponent, withIntermediateDirectories: true)168 let dest = CGImageDestinationCreateWithURL(URL(fileURLWithPath: path) as CFURL, type.identifier as CFString, 1, nil)!169 CGImageDestinationAddImage(dest, img, [kCGImageDestinationLossyCompressionQuality: quality] as CFDictionary)170 CGImageDestinationFinalize(dest)171 }172173 @Test func resizedAndRecompressedCopiesAreGroupedAndUnrelatedImagesAreNot() throws {174 let s = Sandbox()175 let a = Self.picture(seed: 1), b = Self.picture(seed: 2)176 Self.write(a, to: s.path("p/a.jpg"))177 Self.write(a, to: s.path("p/a-small.jpg"), quality: 0.6, scale: 0.4)178 Self.write(a, to: s.path("p/a.png"), type: .png)179 Self.write(b, to: s.path("p/b.jpg"))180 for i in 3 ..< 9 { Self.write(Self.picture(seed: i), to: s.path("p/other\(i).jpg")) }181 var o = SimilarImageOptions(); o.minimumSize = 512182 let r = SimilarImageFinder().run(tree: try s.scan().tree, options: o)183 #expect(r.imagesExamined == 10)184 #expect(r.groups.count == 1)185 let names = Set(r.groups.first?.images.map { ($0.path as NSString).lastPathComponent } ?? [])186 #expect(names == ["a.jpg", "a-small.jpg", "a.png"])187 #expect(r.groups.first?.images.first?.pixelWidth == 640)188 #expect((r.groups.first?.confidence ?? 0) > 0.3)189 }190191 @Test func thoroughModeFindsCrops() throws {192 let s = Sandbox()193 let a = Self.picture(seed: 21)194 Self.write(a, to: s.path("a.jpg"))195 Self.write(a, to: s.path("a-crop.jpg"), crop: 0.06)196 for i in 30 ..< 34 { Self.write(Self.picture(seed: i), to: s.path("o\(i).jpg")) }197 var o = SimilarImageOptions(); o.mode = .thorough198 let r = SimilarImageFinder().run(tree: try s.scan().tree, options: o)199 #expect(r.mode == .thorough)200 #expect(r.groups.count == 1)201 #expect(r.groups.first?.images.count == 2)202 }203204 @Test func corruptImagesAreCountedNotCrashed() throws {205 let s = Sandbox()206 s.file("broken.jpg", size: 50_000, seed: 3)207 Self.write(Self.picture(seed: 5), to: s.path("ok.jpg"))208 let r = SimilarImageFinder().run(tree: try s.scan().tree)209 #expect(r.unreadable == 1)210 #expect(r.groups.isEmpty)211 }212}213