SPB Git

Repository Public License

DuplicateAndImageTests.swift 10.8 KB 213 lines swift

1// ORIX Clean — Packages/ORIXCleanCore/Tests/ORIXCleanCoreTests/DuplicateAndImageTests.swift2// Description: Duplicate detection and similar-image tests on generated files.3// Author: Simon-Pierre Boucher — contact@spboucher.ai4import CoreGraphics5import Darwin6import Foundation7import ImageIO8import Testing9import UniformTypeIdentifiers10@testable import ORIXCleanCore1112@Suite struct DuplicateTests {13    @Test func exactDuplicatesAreGroupedAndDifferentFilesAreNot() throws {14        let s = Sandbox()15        s.file("a/one.bin", size: 200_000, seed: 1)16        s.file("b/one-copy.bin", size: 200_000, seed: 1)17        s.file("c/one-third.bin", size: 200_000, seed: 1)18        s.file("a/same-size-different.bin", size: 200_000, seed: 2)19        s.file("a/small.bin", size: 10, seed: 3)20        s.file("b/small.bin", size: 10, seed: 3)21        s.file("a/unique.bin", size: 12_345, seed: 4)22        let r = DuplicateFinder().run(tree: try s.scan().tree)23        #expect(r.groups.count == 2)24        #expect(r.groups[0].files.count == 3 && r.groups[0].logicalSize == 200_000)25        #expect(r.groups[0].redundantLogical == 400_000)26        #expect(r.groups[1].files.count == 2 && r.groups[1].logicalSize == 10)27        #expect(r.redundantLogical == 400_010)28        #expect(!r.groups[0].sharesStorage)29        let expected = shell("/usr/bin/shasum", ["-a", "256", s.path("a/one.bin")]).output.split(separator: " ").first.map(String.init)30        #expect(r.groups[0].digest == expected)31        var o = DuplicateOptions(); o.minimumSize = 100032        #expect(DuplicateFinder().run(tree: try s.scan().tree, options: o).groups.count == 1)33    }3435    @Test func differencesOutsideTheSampledWindowsAreCaughtByTheFullHash() throws {36        let s = Sandbox()37        let a = s.file("x.bin", size: 3_000_000, seed: 7)38        let b = s.path("y.bin")39        try FileManager.default.copyItem(atPath: a, toPath: b)40        // Flip one byte far from the head, middle and tail windows. The copy is a clone, so write breaks sharing for one block.41        let fd = open(b, O_WRONLY); var byte: UInt8 = 0xFF42        pwrite(fd, &byte, 1, 700_001); close(fd)43        #expect(DuplicateFinder.sampleHash(a, 3_000_000) == DuplicateFinder.sampleHash(b, 3_000_000))44        #expect(DuplicateFinder().run(tree: try s.scan().tree).groups.isEmpty)45        #expect(!DuplicateFinder.identicalContents(a, b))46    }4748    @Test func collisionsInThePreliminaryFilterDoNotCreateFalseGroups() throws {49        let s = Sandbox()50        for i in 0 ..< 6 { s.file("f\(i).bin", size: 100_000, seed: UInt8(i / 2)) }51        let colliding = DuplicateFinder(sampler: { _, _ in Data([1, 2, 3]) })52        let r = colliding.run(tree: try s.scan().tree)53        #expect(r.groups.count == 3)54        #expect(r.groups.allSatisfy { $0.files.count == 2 })55    }5657    @Test func hardLinksAreOneFileNotDuplicates() throws {58        let s = Sandbox()59        let a = s.file("a.bin", size: 80_000, seed: 9)60        link(a, s.path("a-link.bin"))61        #expect(DuplicateFinder().run(tree: try s.scan().tree).groups.isEmpty)62        s.file("copy.bin", size: 80_000, seed: 9)63        let r = DuplicateFinder().run(tree: try s.scan().tree)64        #expect(r.groups.count == 1)65        #expect(r.groups[0].files.count == 2)66        let linked = r.groups[0].files.first { !$0.hardLinkPaths.isEmpty }!67        #expect(linked.hardLinkPaths.count == 1)68        // Removing one name of a hard-linked file frees nothing.69        #expect(r.groups[0].recoverable(removing: [linked.path]).maximum == 0)70    }7172    @Test func clonesAreDuplicatesThatShareStorage() throws {73        let s = Sandbox()74        let a = s.file("orig.bin", size: 2_000_000, seed: 5)75        clonefile(a, s.path("clone.bin"), 0)76        let r = DuplicateFinder().run(tree: try s.scan().tree)77        #expect(r.groups.count == 1)78        #expect(r.groups[0].sharesStorage)79        let est = r.groups[0].recoverable(removing: [s.path("clone.bin")])80        #expect(est.minimum < 100_000)81        #expect(est.maximum >= 2_000_000)82        #expect(!est.isExact)83    }8485    @Test func largeFilesAndUnreadableFiles() throws {86        let s = Sandbox()87        let big = s.file("big1.bin", size: 48_000_000, seed: 11)88        // A real copy (not a clone) so both are read in full.89        let data = try Data(contentsOf: URL(fileURLWithPath: big))90        FileManager.default.createFile(atPath: s.path("big2.bin"), contents: data)91        s.file("locked1.bin", size: 70_000, seed: 12)92        s.file("locked2.bin", size: 70_000, seed: 12)93        chmod(s.path("locked2.bin"), 0o000)94        let finder = DuplicateFinder()95        let r = finder.run(tree: try s.scan().tree)96        #expect(r.groups.count == 1)97        #expect(r.groups[0].logicalSize == 48_000_000)98        #expect(r.bytesHashed >= 96_000_000)99        #expect(r.unreadable == [s.path("locked2.bin")] || r.unreadable.isEmpty)100        #expect(finder.progress.stage == .done)101    }102103    @Test func aFileThatChangedSinceTheScanIsDropped() throws {104        let s = Sandbox()105        s.file("a.bin", size: 50_000, seed: 1)106        s.file("b.bin", size: 50_000, seed: 1)107        let tree = try s.scan().tree108        let h = FileHandle(forWritingAtPath: s.path("b.bin"))!109        h.seekToEndOfFile(); h.write(Data([1, 2, 3])); try h.close()110        #expect(DuplicateFinder().run(tree: tree).groups.isEmpty)111        #expect(DuplicateFinder.fullHash(path: s.path("b.bin"), expectedSize: 50_000) == nil)112        #expect(DuplicateFinder.fullHash(path: s.path("b.bin")) != nil)113    }114115    @Test func packageContentsAreSkippedUnlessAsked() throws {116        let s = Sandbox()117        s.file("One.app/Contents/Resources/asset.bin", size: 40_000, seed: 3)118        s.file("Two.app/Contents/Resources/asset.bin", size: 40_000, seed: 3)119        let tree = try s.scan().tree120        #expect(DuplicateFinder().run(tree: tree).groups.isEmpty)121        var o = DuplicateOptions(); o.includePackageContents = true122        #expect(DuplicateFinder().run(tree: tree, options: o).groups.count == 1)123    }124125    @Test func cancellationReturnsNoGroups() throws {126        let s = Sandbox()127        for i in 0 ..< 20 { s.file("f\(i).bin", size: 300_000, seed: 1) }128        let f = DuplicateFinder()129        f.cancel()130        let r = f.run(tree: try s.scan().tree)131        #expect(r.cancelled && r.groups.isEmpty)132    }133}134135@Suite struct SimilarImageTests {136    /// A deterministic picture: gradient background and a few shapes that depend on `seed`.137    static func picture(seed: Int, size: Int = 640) -> CGImage {138        let ctx = CGContext(data: nil, width: size, height: size, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpaceCreateDeviceRGB(),139                            bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue)!140        var x = UInt64(seed) &* 6364136223846793005 &+ 1442695040888963407141        func rnd() -> CGFloat { x = x &* 6364136223846793005 &+ 1442695040888963407; return CGFloat((x >> 33) % 1000) / 1000 }142        let colors = [CGColor(red: rnd(), green: rnd(), blue: rnd(), alpha: 1), CGColor(red: rnd(), green: rnd(), blue: rnd(), alpha: 1)] as CFArray143        let g = CGGradient(colorsSpace: CGColorSpaceCreateDeviceRGB(), colors: colors, locations: [0, 1])!144        ctx.drawLinearGradient(g, start: .zero, end: CGPoint(x: CGFloat(size) * rnd(), y: CGFloat(size)), options: [.drawsAfterEndLocation])145        for _ in 0 ..< 14 {146            ctx.setFillColor(CGColor(red: rnd(), green: rnd(), blue: rnd(), alpha: 1))147            let r = CGRect(x: rnd() * CGFloat(size), y: rnd() * CGFloat(size), width: rnd() * CGFloat(size) / 2.5 + 20, height: rnd() * CGFloat(size) / 2.5 + 20)148            if rnd() > 0.5 { ctx.fillEllipse(in: r) } else { ctx.fill(r) }149        }150        return ctx.makeImage()!151    }152153    static func write(_ image: CGImage, to path: String, type: UTType = .jpeg, quality: Double = 0.9, scale: CGFloat = 1, crop: CGFloat = 0) {154        var img = image155        if crop > 0 {156            let inset = CGFloat(image.width) * crop157            img = image.cropping(to: CGRect(x: inset, y: inset, width: CGFloat(image.width) - 2 * inset, height: CGFloat(image.height) - 2 * inset))!158        }159        if scale != 1 {160            let w = Int(CGFloat(img.width) * scale), h = Int(CGFloat(img.height) * scale)161            let ctx = CGContext(data: nil, width: w, height: h, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpaceCreateDeviceRGB(),162                                bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue)!163            ctx.interpolationQuality = .high164            ctx.draw(img, in: CGRect(x: 0, y: 0, width: w, height: h))165            img = ctx.makeImage()!166        }167        try? FileManager.default.createDirectory(atPath: (path as NSString).deletingLastPathComponent, withIntermediateDirectories: true)168        let dest = CGImageDestinationCreateWithURL(URL(fileURLWithPath: path) as CFURL, type.identifier as CFString, 1, nil)!169        CGImageDestinationAddImage(dest, img, [kCGImageDestinationLossyCompressionQuality: quality] as CFDictionary)170        CGImageDestinationFinalize(dest)171    }172173    @Test func resizedAndRecompressedCopiesAreGroupedAndUnrelatedImagesAreNot() throws {174        let s = Sandbox()175        let a = Self.picture(seed: 1), b = Self.picture(seed: 2)176        Self.write(a, to: s.path("p/a.jpg"))177        Self.write(a, to: s.path("p/a-small.jpg"), quality: 0.6, scale: 0.4)178        Self.write(a, to: s.path("p/a.png"), type: .png)179        Self.write(b, to: s.path("p/b.jpg"))180        for i in 3 ..< 9 { Self.write(Self.picture(seed: i), to: s.path("p/other\(i).jpg")) }181        var o = SimilarImageOptions(); o.minimumSize = 512182        let r = SimilarImageFinder().run(tree: try s.scan().tree, options: o)183        #expect(r.imagesExamined == 10)184        #expect(r.groups.count == 1)185        let names = Set(r.groups.first?.images.map { ($0.path as NSString).lastPathComponent } ?? [])186        #expect(names == ["a.jpg", "a-small.jpg", "a.png"])187        #expect(r.groups.first?.images.first?.pixelWidth == 640)188        #expect((r.groups.first?.confidence ?? 0) > 0.3)189    }190191    @Test func thoroughModeFindsCrops() throws {192        let s = Sandbox()193        let a = Self.picture(seed: 21)194        Self.write(a, to: s.path("a.jpg"))195        Self.write(a, to: s.path("a-crop.jpg"), crop: 0.06)196        for i in 30 ..< 34 { Self.write(Self.picture(seed: i), to: s.path("o\(i).jpg")) }197        var o = SimilarImageOptions(); o.mode = .thorough198        let r = SimilarImageFinder().run(tree: try s.scan().tree, options: o)199        #expect(r.mode == .thorough)200        #expect(r.groups.count == 1)201        #expect(r.groups.first?.images.count == 2)202    }203204    @Test func corruptImagesAreCountedNotCrashed() throws {205        let s = Sandbox()206        s.file("broken.jpg", size: 50_000, seed: 3)207        Self.write(Self.picture(seed: 5), to: s.path("ok.jpg"))208        let r = SimilarImageFinder().run(tree: try s.scan().tree)209        #expect(r.unreadable == 1)210        #expect(r.groups.isEmpty)211    }212}213
Files main
Menu

Theme