SPB Git

Repository Public License

AnalyzerTests.swift 28.9 KB 581 lines swift

1// ORIX Secure2// File: Packages/ORIXMediaCleanup/Tests/ORIXMediaCleanupTests/AnalyzerTests.swift3// Description: Grouping engine: duplicates, complete-linkage similarity, time windows, global pass, keeper rules, bursts,4//              lists, recoverable-bytes arithmetic, determinism and cancellation.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import ORIXSecureModel10import Testing11@testable import ORIXMediaCleanup1213@Suite("Cleanup analyzer")14struct AnalyzerTests {15    let base: UInt64 = 0x5A5A_F00F_1234_98761617    // MARK: Similarity1819    @Test("A slowly drifting sequence is not chained into one group")20    func noChaining() async throws {21        // Photo i differs from photo i+1 by 3 bits and from photo i+k by 3k bits (both hashes).22        var items: [MediaItem] = []23        var hashes: [String: UInt64] = [:]24        var hash = base25        for index in 0..<20 {26            if index > 0 { for bit in 0..<3 { hash ^= 1 << UInt64((index - 1) * 3 + bit) } }27            let id = String(format: "p%02d", index)28            hashes[id] = hash29            items.append(Fixture.photo(id, at: Double(index) * 10, fingerprint: Fixture.fingerprint(d: hash)))30        }31        #expect(ImageFingerprint.hammingDistance(hashes["p00"]!, hashes["p19"]!) == 57)32        let analysis = try await CleanupAnalyzer().analyze(items)33        let groups = analysis.groups(in: .similar)34        #expect(groups.count >= 6)35        var grouped = 036        for group in groups {37            grouped += group.items.count38            #expect(group.items.count <= 3)39            for a in group.items {40                for b in group.items {41                    let distance = ImageFingerprint.hammingDistance(hashes[a.id]!, hashes[b.id]!)42                    #expect(distance <= 8)43                }44            }45        }46        #expect(grouped >= 18)47        // Single linkage would have produced one group of 20.48        let together = groups.contains { group in group.items.contains { $0.id == "p00" } && group.items.contains { $0.id == "p05" } }49        #expect(!together)50    }5152    @Test("Time window: sessions are runs of photos at most 3 minutes apart")53    func timeWindow() async throws {54        let print = Fixture.fingerprint(d: base)55        let other = Fixture.fingerprint(d: ~base)56        let near = [Fixture.photo("a", at: 0, fingerprint: print), Fixture.photo("b", at: 100, fingerprint: print)]57        let nearGroups = try await CleanupAnalyzer().analyze(near).groups(in: .similar)58        #expect(Fixture.groupIDs(nearGroups) == [["a", "b"]])5960        let far = [Fixture.photo("a", at: 0, fingerprint: print), Fixture.photo("b", at: 200, fingerprint: print)]61        let farAnalysis = try await CleanupAnalyzer().analyze(far)62        #expect(farAnalysis.groups(in: .similar).isEmpty)6364        // A photo in between keeps the session open.65        let bridged = far + [Fixture.photo("m", at: 100, fingerprint: other)]66        let bridgedGroups = try await CleanupAnalyzer().analyze(bridged).groups(in: .similar)67        #expect(Fixture.groupIDs(bridgedGroups) == [["a", "b"]])6869        var wide = CleanupConfiguration()70        wide.similarTimeWindow = 60071        let wideGroups = try await CleanupAnalyzer(configuration: wide).analyze(far).groups(in: .similar)72        #expect(Fixture.groupIDs(wideGroups) == [["a", "b"]])73    }7475    @Test("Lookback closes a group after too many later photos in the same session")76    func lookback() async throws {77        var config = CleanupConfiguration()78        config.similarLookback = 579        var items = [Fixture.photo("a", at: 0, fingerprint: Fixture.fingerprint(d: base))]80        for index in 0..<10 {81            items.append(Fixture.photo("f\(index)", at: Double(index + 1) * 10, fingerprint: Fixture.fingerprint(d: Fixture.flipping(base, bits: 32, seed: UInt64(index)))))82        }83        items.append(Fixture.photo("z", at: 120, fingerprint: Fixture.fingerprint(d: base)))84        let closed = try await CleanupAnalyzer(configuration: config).analyze(items)85        #expect(closed.groups(in: .similar).isEmpty)86        let open = try await CleanupAnalyzer().analyze(items)87        #expect(Fixture.groupIDs(open.groups(in: .similar)) == [["a", "z"]])88    }8990    @Test("Global pass: off by default, strict thresholds across sessions")91    func globalPass() async throws {92        let near = Fixture.flipping(base, bits: 5, seed: 1)93        let items = [94            Fixture.photo("a", at: 0, fingerprint: Fixture.fingerprint(d: base)),95            Fixture.photo("b", at: 50_000, fingerprint: Fixture.fingerprint(d: base)),96            Fixture.photo("c", at: 100_000, fingerprint: Fixture.fingerprint(d: near)),97            Fixture.photo("d", at: 100_030, fingerprint: Fixture.fingerprint(d: Fixture.flipping(near, bits: 2, seed: 2))),98            Fixture.photo("e", at: 150_000, fingerprint: Fixture.fingerprint(d: Fixture.flipping(base, bits: 2, seed: 3))),99        ]100        let off = try await CleanupAnalyzer().analyze(items)101        #expect(Fixture.groupIDs(off.groups(in: .similar)) == [["c", "d"]])102103        var config = CleanupConfiguration()104        config.globalPassEnabled = true105        let on = try await CleanupAnalyzer(configuration: config).analyze(items)106        // a, b (distance 0) and e (distance 2) meet across sessions; c is 5 bits away: too far for the strict pass.107        #expect(Fixture.groupIDs(on.groups(in: .similar)) == [["a", "b", "e"], ["c", "d"]])108109        config.globalDHashThreshold = 6110        config.globalPHashThreshold = 6111        let loose = try await CleanupAnalyzer(configuration: config).analyze(items)112        let merged = loose.groups(in: .similar).contains { group in group.items.contains { $0.id == "a" } && group.items.contains { $0.id == "c" } }113        #expect(merged)114    }115116    @Test("Colour and luminance reject same-structure photos")117    func colourAndLuma() async throws {118        var red = Fixture.plainSignature119        var blue = Fixture.plainSignature120        for cell in 0..<16 {121            red[cell * 3] = 220122            blue[cell * 3 + 2] = 220123        }124        let colours = [125            Fixture.photo("r", at: 0, fingerprint: Fixture.fingerprint(d: base, signature: red)),126            Fixture.photo("b", at: 10, fingerprint: Fixture.fingerprint(d: base, signature: blue)),127        ]128        let colourAnalysis = try await CleanupAnalyzer().analyze(colours)129        #expect(colourAnalysis.groups(in: .similar).isEmpty)130        let lumas = [131            Fixture.photo("dark", at: 0, fingerprint: Fixture.fingerprint(d: base, luma: 30)),132            Fixture.photo("light", at: 10, fingerprint: Fixture.fingerprint(d: base, luma: 200)),133        ]134        let lumaAnalysis = try await CleanupAnalyzer().analyze(lumas)135        #expect(lumaAnalysis.groups(in: .similar).isEmpty)136    }137138    @Test("Videos and screenshots stay out of visual similarity by default")139    func exclusions() async throws {140        let print = Fixture.fingerprint(d: base)141        var video = Fixture.video("v", at: 5, duration: 10)142        video.fingerprint = print143        let items = [144            Fixture.photo("a", at: 0, fingerprint: print), video,145            Fixture.photo("s1", at: 10, fingerprint: print, screenshot: true),146            Fixture.photo("s2", at: 20, fingerprint: print, screenshot: true),147        ]148        let analysis = try await CleanupAnalyzer().analyze(items)149        #expect(analysis.groups(in: .similar).isEmpty)150        var config = CleanupConfiguration()151        config.includeScreenshotsInSimilarity = true152        let included = try await CleanupAnalyzer(configuration: config).analyze(items)153        #expect(Fixture.groupIDs(included.groups(in: .similar)) == [["a", "s1", "s2"]])154    }155156    // MARK: Keeper157158    struct KeeperCase: Sendable, CustomTestStringConvertible {159        var name: String160        var items: [MediaItem]161        var keeper: String162        var rule: KeeperRule163        var testDescription: String { name }164    }165166    static let keeperCases: [KeeperCase] = [167        KeeperCase(name: "favourite beats edited, resolution, size and date", items: [168            Fixture.photo("a", at: 0, width: 4000, height: 3000, size: 900, edited: true),169            Fixture.photo("b", at: 9, width: 100, height: 100, size: 1, favorite: true),170        ], keeper: "b", rule: .favorite),171        KeeperCase(name: "edited beats resolution", items: [172            Fixture.photo("a", at: 0, width: 4000, height: 3000),173            Fixture.photo("b", at: 9, width: 2000, height: 1500, edited: true),174        ], keeper: "b", rule: .edited),175        KeeperCase(name: "highest resolution", items: [176            Fixture.photo("a", at: 0, width: 2000, height: 1500, size: 999),177            Fixture.photo("b", at: 9, width: 4000, height: 3000, size: 1),178        ], keeper: "b", rule: .highestResolution),179        KeeperCase(name: "largest measured size", items: [180            Fixture.photo("a", at: 0, size: 100),181            Fixture.photo("b", at: 9, size: 300),182            Fixture.photo("c", at: 5, size: 200),183        ], keeper: "b", rule: .largestMeasuredSize),184        KeeperCase(name: "size is ignored when one size is unknown; earliest wins", items: [185            Fixture.photo("a", at: 5, size: 100),186            Fixture.photo("b", at: 9, size: 300),187            Fixture.photo("c", at: 1),188        ], keeper: "c", rule: .earliest),189        KeeperCase(name: "two favourites: the rule that separates them is reported", items: [190            Fixture.photo("a", at: 0, width: 100, height: 100, favorite: true),191            Fixture.photo("b", at: 9, width: 200, height: 200, favorite: true),192            Fixture.photo("c", at: 0, width: 900, height: 900),193        ], keeper: "b", rule: .highestResolution),194        KeeperCase(name: "full tie falls back to identifier order", items: [195            Fixture.photo("b", at: 0), Fixture.photo("a", at: 0),196        ], keeper: "a", rule: .identifierOrder),197        KeeperCase(name: "undated items lose to dated ones", items: [198            Fixture.photo("a", at: nil), Fixture.photo("b", at: 50),199        ], keeper: "b", rule: .earliest),200    ]201202    @Test("Keeper rules", arguments: keeperCases)203    func keeperRules(_ testCase: KeeperCase) {204        let forward = CleanupAnalyzer.chooseKeeper(among: testCase.items)205        #expect(forward?.keeper.id == testCase.keeper)206        #expect(forward?.rule == testCase.rule)207        let reversed = CleanupAnalyzer.chooseKeeper(among: Array(testCase.items.reversed()))208        #expect(reversed?.keeper.id == testCase.keeper)209        #expect(reversed?.rule == testCase.rule)210    }211212    @Test("Keeper of an empty list is nil; every rule has an explanation")213    func keeperEdges() {214        #expect(CleanupAnalyzer.chooseKeeper(among: []) == nil)215        for rule in KeeperRule.allCases { #expect(!rule.explanation.isEmpty) }216    }217218    // MARK: Duplicates219220    @Test("Exact duplicates by content hash: measured, non-keepers pre-selected")221    func exactDuplicates() async throws {222        let items = [223            Fixture.photo("a", at: 0, size: 1000, hash: "h1"),224            Fixture.photo("b", at: 5000, size: 1000, hash: "h1", favorite: true),225            Fixture.photo("c", at: 9000, size: 1000, hash: "h1"),226            Fixture.photo("d", at: 10, size: 500, hash: "h2"),227            Fixture.photo("e", at: 20, size: 700, hash: nil),228        ]229        let analysis = try await CleanupAnalyzer().analyze(items)230        let groups = analysis.groups(in: .exactDuplicates)231        #expect(groups.count == 1)232        let group = try #require(groups.first)233        #expect(group.confidence == .measured)234        #expect(group.keeperID == "b")235        #expect(group.keeperRule == .favorite)236        #expect(group.preselectedIDs.sorted() == ["a", "c"])237        #expect(group.candidateIDs.sorted() == ["a", "c"])238        #expect(group.recoverable == .measured(2000))239        #expect(group.measuredPortion == 2000)240        #expect(analysis.preselectedIDs == ["a", "c"])241        #expect(analysis.preselectedTotals == CleanupTotals(candidateCount: 2, measuredBytes: 2000, measuredCount: 2, notMeasuredCount: 0))242        #expect(analysis.groups(in: .probableDuplicates).isEmpty)243        #expect(analysis.recentlyDeletedNote.contains("Recently Deleted"))244    }245246    @Test("Probable duplicates: heuristic, separate category, nothing pre-selected")247    func probableDuplicates() async throws {248        let print = Fixture.fingerprint(d: base, p: ~base)249        let items = [250            Fixture.photo("a", at: 100.2, fingerprint: print),251            Fixture.photo("b", at: 100.7, fingerprint: print),252            Fixture.photo("c", at: 101.1, fingerprint: print),                       // next second253            Fixture.photo("d", at: 100.4, fingerprint: print, width: 1000, height: 800), // other dimensions254        ]255        let analysis = try await CleanupAnalyzer().analyze(items)256        let groups = analysis.groups(in: .probableDuplicates)257        #expect(Fixture.groupIDs(groups) == [["a", "b"]])258        #expect(groups.first?.confidence == .heuristic)259        #expect(groups.first?.preselectedIDs.isEmpty == true)260        #expect(analysis.preselectedIDs.isEmpty)261        #expect(analysis.groups(in: .exactDuplicates).isEmpty)262        // The keeper of the probable pair still takes part in visual similarity with c and d; the other copy does not.263        let similar = analysis.groups(in: .similar) + analysis.groups(in: .burstLike)264        #expect(Fixture.groupIDs(similar) == [["a", "c", "d"]])265    }266267    @Test("Hashed items with different content are not probable duplicates")268    func hashedAreNotProbable() async throws {269        let print = Fixture.fingerprint(d: base)270        let items = [271            Fixture.photo("a", at: 100.2, fingerprint: print, hash: "x"),272            Fixture.photo("b", at: 100.7, fingerprint: print, hash: "y"),273        ]274        let analysis = try await CleanupAnalyzer().analyze(items)275        #expect(analysis.groups(in: .probableDuplicates).isEmpty)276        #expect(Fixture.groupIDs(analysis.groups(in: .similar)) == [["a", "b"]])277    }278279    @Test("Recoverable bytes: unknown sizes make the amount unavailable, with the measured portion kept")280    func recoverableArithmetic() async throws {281        let items = [282            Fixture.photo("a", at: 0, size: 5000, hash: "h", favorite: true),283            Fixture.photo("b", at: 1, size: 3000, hash: "h"),284            Fixture.photo("c", at: 2, size: nil, hash: "h"),285            Fixture.photo("d", at: 3, size: 1500, hash: "h"),286        ]287        let analysis = try await CleanupAnalyzer().analyze(items)288        let group = try #require(analysis.groups(in: .exactDuplicates).first)289        #expect(group.recoverable.value == nil)290        #expect(group.recoverable.unavailability?.kind == .notRequested)291        #expect(group.measuredPortion == 4500)292        #expect(group.unmeasuredCount == 1)293        let totals = try #require(analysis.category(.exactDuplicates)?.totals)294        #expect(totals == CleanupTotals(candidateCount: 3, measuredBytes: 4500, measuredCount: 2, notMeasuredCount: 1))295        #expect(totals.recoverable.value == nil)296        #expect(analysis.totals == totals)297        // The keeper's size never counts.298        let sizes = CleanupAnalyzer.recoverable(of: [])299        #expect(sizes.reading == .measured(0))300    }301302    // MARK: Bursts303304    @Test("Real bursts, burst-like captures and plain similar groups are told apart")305    func bursts() async throws {306        let one = Fixture.fingerprint(d: base)307        let two = Fixture.fingerprint(d: Fixture.flipping(base, bits: 30, seed: 5))308        let three = Fixture.fingerprint(d: Fixture.flipping(base, bits: 30, seed: 6))309        let four = Fixture.fingerprint(d: Fixture.flipping(base, bits: 30, seed: 7))310        let items = [311            // camera burst312            Fixture.photo("r1", at: 0, fingerprint: one, burst: "B1"),313            Fixture.photo("r2", at: 0.1, fingerprint: one, burst: "B1"),314            Fixture.photo("r3", at: 0.2, fingerprint: one, favorite: true, burst: "B1"),315            // burst-like: four similar photos within two seconds of each other316            Fixture.photo("l1", at: 1000, fingerprint: two, width: 100, height: 100),317            Fixture.photo("l2", at: 1001.5, fingerprint: two, width: 100, height: 100),318            Fixture.photo("l3", at: 1003, fingerprint: two, width: 100, height: 100),319            Fixture.photo("l4", at: 1004.9, fingerprint: two, width: 100, height: 100),320            // similar, but too slow to be burst-like321            Fixture.photo("s1", at: 2000, fingerprint: three),322            Fixture.photo("s2", at: 2030, fingerprint: three),323            Fixture.photo("s3", at: 2060, fingerprint: three),324            // fast, but only two325            Fixture.photo("t1", at: 3000, fingerprint: four),326            Fixture.photo("t2", at: 3001, fingerprint: four),327            // a lone burst member is not a burst328            Fixture.photo("x", at: 4000, fingerprint: one, burst: "B2"),329        ]330        let analysis = try await CleanupAnalyzer().analyze(items)331        let real = analysis.groups(in: .bursts)332        #expect(Fixture.groupIDs(real) == [["r1", "r2", "r3"]])333        #expect(real.first?.id == "bursts:B1")334        #expect(real.first?.keeperID == "r3")335        #expect(real.first?.confidence == Confidence.measured)336        #expect(real.first?.preselectedIDs.isEmpty == true)337        #expect(Fixture.groupIDs(analysis.groups(in: .burstLike)) == [["l1", "l2", "l3", "l4"]])338        #expect(analysis.groups(in: .burstLike).first?.confidence == Confidence.heuristic)339        #expect(Fixture.groupIDs(analysis.groups(in: .similar)) == [["s1", "s2", "s3"], ["t1", "t2"]])340    }341342    // MARK: Lists343344    @Test("Screenshots, old screenshots, videos and Live Photos")345    func lists() async throws {346        var config = CleanupConfiguration()347        config.oldScreenshotAgeDays = 30348        config.referenceDate = Fixture.date(100 * 86_400)349        let items = [350            Fixture.photo("shot-old", at: 10 * 86_400, size: 400, screenshot: true),351            Fixture.photo("shot-new", at: 90 * 86_400, screenshot: true),352            Fixture.video("big", at: 0, duration: 30, size: 900_000_000),353            Fixture.video("bigger", at: 1, duration: 20, size: 2_000_000_000),354            Fixture.video("small", at: 2, duration: 5, size: 3_000_000),355            Fixture.video("unknown-4k-long", at: 3, duration: 600, width: 3840, height: 2160),356            Fixture.video("unknown-hd-2min", at: 4, duration: 130),357            Fixture.video("unknown-short", at: 5, duration: 4),358            Fixture.photo("live", at: 50, size: 5_000_000, kind: .livePhoto),359        ]360        let analysis = try await CleanupAnalyzer(configuration: config).analyze(items)361362        #expect(Fixture.groupIDs(analysis.groups(in: .screenshots)) == [["shot-new", "shot-old"]])363        #expect(Fixture.groupIDs(analysis.groups(in: .oldScreenshots)) == [["shot-old"]])364        let shots = try #require(analysis.groups(in: .screenshots).first)365        #expect(shots.keeperID == nil)366        #expect(shots.preselectedIDs.isEmpty)367        #expect(shots.recoverable.value == nil)368        #expect(shots.measuredPortion == 400)369370        let large = analysis.groups(in: .largeVideos)371        #expect(large.count == 2)372        let measured = try #require(large.first { $0.ordering == .byMeasuredSize })373        #expect(measured.items.map { $0.id } == ["bigger", "big"])374        #expect(measured.recoverable == .measured(2_900_000_000))375        #expect(measured.confidence == .measured)376        #expect(measured.sizeEstimates.isEmpty)377        let estimated = try #require(large.first { $0.ordering == .byEstimate })378        #expect(estimated.items.map { $0.id } == ["unknown-4k-long", "unknown-hd-2min"])379        #expect(estimated.confidence == .heuristic)380        #expect(estimated.sizeEstimates == ["unknown-4k-long": .probablyVeryLarge, "unknown-hd-2min": .probablyLarge])381        // An estimate is never turned into bytes.382        #expect(estimated.recoverable.value == nil)383        #expect(estimated.measuredPortion == 0)384        #expect(estimated.unmeasuredCount == 2)385        for estimate in VideoSizeEstimate.allCases { #expect(estimate.title.contains("estimate")) }386387        #expect(analysis.groups(in: .longVideos).first?.items.map { $0.id } == ["unknown-4k-long", "unknown-hd-2min"])388389        let live = try #require(analysis.groups(in: .livePhotos).first)390        #expect(live.isInformational)391        #expect(live.candidateIDs.isEmpty)392        #expect(analysis.category(.livePhotos)?.totals.candidateCount == 0)393394        // Totals count each candidate once, although shot-old and the long videos sit in two categories.395        #expect(analysis.totals.candidateCount == 6)396        #expect(analysis.totals.measuredBytes == 2_900_000_400)397        #expect(analysis.totals.notMeasuredCount == 3)398        #expect(analysis.preselectedIDs.isEmpty)399        #expect(analysis.statistics.videoCount == 6)400        #expect(analysis.statistics.livePhotoCount == 1)401        #expect(analysis.categories.map { $0.kind } == [.screenshots, .oldScreenshots, .largeVideos, .longVideos, .livePhotos])402    }403404    @Test("Video size estimate buckets")405    func estimates() {406        #expect(VideoSizeEstimate.estimate(duration: 5, pixelWidth: 1920, pixelHeight: 1080) == .probablySmall)407        #expect(VideoSizeEstimate.estimate(duration: 40, pixelWidth: 1920, pixelHeight: 1080) == .probablyMedium)408        #expect(VideoSizeEstimate.estimate(duration: 130, pixelWidth: 1920, pixelHeight: 1080) == .probablyLarge)409        #expect(VideoSizeEstimate.estimate(duration: 600, pixelWidth: 1920, pixelHeight: 1080) == .probablyVeryLarge)410        #expect(VideoSizeEstimate.estimate(duration: -1, pixelWidth: -5, pixelHeight: 10) == .probablySmall)411    }412413    // MARK: Statistics, determinism, cancellation414415    @Test("Statistics: scanned, fingerprinted, skipped with reasons")416    func statistics() async throws {417        let print = Fixture.fingerprint(d: base)418        let items = [419            Fixture.photo("a", at: 0, fingerprint: print, size: 10, hash: "h"),420            Fixture.photo("a", at: 0, fingerprint: print),421            Fixture.photo("nofp", at: 5),422            Fixture.photo("nodate", at: nil, fingerprint: print),423            Fixture.video("v", at: 9, duration: 3),424        ]425        let analysis = try await CleanupAnalyzer().analyze(items)426        let stats = analysis.statistics427        #expect(stats.itemsScanned == 4)428        #expect(stats.itemsFingerprinted == 2)429        #expect(stats.itemsMeasured == 1)430        #expect(stats.itemsHashed == 1)431        #expect(stats.photoCount == 3)432        #expect(stats.skipped == [433            SkippedCount(reason: .duplicateIdentifier, count: 1),434            SkippedCount(reason: .noFingerprint, count: 1),435            SkippedCount(reason: .noCreationDate, count: 1),436        ])437        #expect(stats.duration >= 0)438        let empty = try await CleanupAnalyzer().analyze([])439        #expect(empty.categories.isEmpty)440        #expect(empty.totals == CleanupTotals())441    }442443    @Test("Deterministic: shuffled input gives the same groups")444    func determinism() async throws {445        let items = SyntheticLibrary.items(count: 3000, seed: 11)446        var config = CleanupConfiguration()447        config.globalPassEnabled = true448        config.oldScreenshotAgeDays = 10449        config.referenceDate = Fixture.date(1_000_000)450        let analyzer = CleanupAnalyzer(configuration: config)451        let reference = try await analyzer.analyze(items)452        #expect(!reference.groups(in: .similar).isEmpty)453        #expect(!reference.groups(in: .exactDuplicates).isEmpty)454        var rng = SeededGenerator(seed: 5)455        for _ in 0..<4 {456            let shuffled = items.shuffled(using: &rng)457            let again = try await analyzer.analyze(shuffled)458            #expect(again.categories == reference.categories)459            #expect(again.totals == reference.totals)460        }461    }462463    @Test("Progress is reported and ends with `finished`")464    func progress() async throws {465        let phases = PhaseRecorder()466        _ = try await CleanupAnalyzer().analyze(SyntheticLibrary.items(count: 2000, seed: 3)) { phases.record($0) }467        let seen = phases.phases468        #expect(seen.first == .preparing)469        #expect(seen.last == .finished)470        #expect(seen.contains(.similarity))471    }472473    @Test("Cancellation stops the analysis promptly")474    func cancellation() async throws {475        let items = SyntheticLibrary.items(count: 50_000, seed: 21)476        let clock = ContinuousClock()477        let task = Task { try await CleanupAnalyzer().analyze(items) }478        try await Task.sleep(for: .milliseconds(20))479        let cancelledAt = clock.now480        task.cancel()481        let result = await task.result482        let waited = clock.now - cancelledAt483        switch result {484        case .success: Issue.record("analysis finished before the cancellation was seen")485        case .failure(let error): #expect(error is CancellationError)486        }487        print("CANCELLATION analysis returned \(waited) after cancel")488        #expect(waited < .seconds(2))489    }490491    @Test("Measurement candidates: group members first, then the longest videos, only unmeasured ones")492    func measurementCandidates() async throws {493        let print = Fixture.fingerprint(d: base)494        let items = [495            Fixture.photo("a", at: 0, fingerprint: print),496            Fixture.photo("b", at: 10, fingerprint: print, size: 5, hash: "known"),497            Fixture.photo("lonely", at: 5000, fingerprint: Fixture.fingerprint(d: ~base)),498            Fixture.video("v-long", at: 1, duration: 500),499            Fixture.video("v-mid", at: 2, duration: 50),500            Fixture.video("v-short", at: 3, duration: 5),501            Fixture.video("v-measured", at: 4, duration: 900, size: 77),502        ]503        let analyzer = CleanupAnalyzer()504        let analysis = try await analyzer.analyze(items)505        let candidates = analyzer.measurementCandidates(in: analysis, items: items, topVideosByDuration: 3)506        #expect(candidates == ["a", "v-long", "v-mid"])507    }508509    @Test("Analysis is Codable")510    func codable() async throws {511        let analysis = try await CleanupAnalyzer().analyze(SyntheticLibrary.items(count: 300, seed: 2))512        let data = try JSONEncoder().encode(analysis)513        let decoded = try JSONDecoder().decode(CleanupAnalysis.self, from: data)514        #expect(decoded == analysis)515    }516}517518final class PhaseRecorder: @unchecked Sendable {519    private let lock = NSLock()520    private var storage: [CleanupProgress.Phase] = []521522    func record(_ progress: CleanupProgress) {523        lock.lock()524        if storage.last != progress.phase { storage.append(progress.phase) }525        lock.unlock()526    }527528    var phases: [CleanupProgress.Phase] {529        lock.lock()530        defer { lock.unlock() }531        return storage532    }533}534535/// A synthetic library built directly from fingerprints: sessions of photos, near-duplicate runs, exact duplicates, bursts,536/// screenshots and videos.537enum SyntheticLibrary {538    static func items(count: Int, seed: UInt64) -> [MediaItem] {539        var rng = SeededGenerator(seed: seed)540        var items: [MediaItem] = []541        items.reserveCapacity(count)542        var time: TimeInterval = 0543        var previous: ImageFingerprint?544        for index in 0..<count {545            // Mostly short gaps (one session), sometimes a long pause.546            time += rng.next() % 40 == 0 ? Double(rng.next() % 5000 + 200) : Double(rng.next() % 30) + 0.25547            let id = String(format: "item-%06d", index)548            let roll = rng.next() % 100549            if roll < 4 {550                items.append(Fixture.video(id, at: time, duration: Double(rng.next() % 400), size: roll < 2 ? Int64(rng.next() % 800_000_000) : nil))551                continue552            }553            var fingerprint: ImageFingerprint554            if roll < 24, let last = previous {555                // A near-duplicate of the previous photo: a few bits away.556                fingerprint = last557                fingerprint.dHash = Fixture.flipping(last.dHash, bits: Int(rng.next() % 4), seed: rng.next())558                fingerprint.pHash = Fixture.flipping(last.pHash, bits: Int(rng.next() % 4), seed: rng.next())559            } else {560                var signature = [UInt8](repeating: 0, count: ImageFingerprint.colorSignatureLength)561                for position in signature.indices { signature[position] = UInt8(truncatingIfNeeded: rng.next()) }562                fingerprint = ImageFingerprint(dHash: rng.next(), pHash: rng.next() & ~1, meanLuma: UInt8(truncatingIfNeeded: rng.next()), colorSignature: signature)563            }564            previous = fingerprint565            var item = Fixture.photo(id, at: time, fingerprint: fingerprint, size: roll % 3 == 0 ? nil : Int64(rng.next() % 6_000_000 + 500_000))566            if roll >= 24 && roll < 27, index > 10 {567                // Byte-identical copy of an earlier photo.568                item.contentHash = "hash-\(index - 7)"569            } else if roll % 5 == 0 {570                item.contentHash = "hash-\(index)"571            }572            if roll >= 30 && roll < 36 { item.isScreenshot = true }573            if roll >= 40 && roll < 44 { item.burstIdentifier = "burst-\(index / 6)"; item.isBurstMember = true }574            if roll == 50 { item.kind = .livePhoto }575            if roll == 60 { item.isFavorite = true }576            items.append(item)577        }578        return items579    }580}581
Files main
Menu

Theme