Repository Public License
1// ORIX Secure2// File: Packages/ORIXMediaCleanup/Tests/ORIXMediaCleanupTests/AnalyzerTests.swift3// Description: Grouping engine: duplicates, complete-linkage similarity, time windows, global pass, keeper rules, bursts,4// lists, recoverable-bytes arithmetic, determinism and cancellation.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import ORIXSecureModel10import Testing11@testable import ORIXMediaCleanup1213@Suite("Cleanup analyzer")14struct AnalyzerTests {15 let base: UInt64 = 0x5A5A_F00F_1234_98761617 // MARK: Similarity1819 @Test("A slowly drifting sequence is not chained into one group")20 func noChaining() async throws {21 // Photo i differs from photo i+1 by 3 bits and from photo i+k by 3k bits (both hashes).22 var items: [MediaItem] = []23 var hashes: [String: UInt64] = [:]24 var hash = base25 for index in 0..<20 {26 if index > 0 { for bit in 0..<3 { hash ^= 1 << UInt64((index - 1) * 3 + bit) } }27 let id = String(format: "p%02d", index)28 hashes[id] = hash29 items.append(Fixture.photo(id, at: Double(index) * 10, fingerprint: Fixture.fingerprint(d: hash)))30 }31 #expect(ImageFingerprint.hammingDistance(hashes["p00"]!, hashes["p19"]!) == 57)32 let analysis = try await CleanupAnalyzer().analyze(items)33 let groups = analysis.groups(in: .similar)34 #expect(groups.count >= 6)35 var grouped = 036 for group in groups {37 grouped += group.items.count38 #expect(group.items.count <= 3)39 for a in group.items {40 for b in group.items {41 let distance = ImageFingerprint.hammingDistance(hashes[a.id]!, hashes[b.id]!)42 #expect(distance <= 8)43 }44 }45 }46 #expect(grouped >= 18)47 // Single linkage would have produced one group of 20.48 let together = groups.contains { group in group.items.contains { $0.id == "p00" } && group.items.contains { $0.id == "p05" } }49 #expect(!together)50 }5152 @Test("Time window: sessions are runs of photos at most 3 minutes apart")53 func timeWindow() async throws {54 let print = Fixture.fingerprint(d: base)55 let other = Fixture.fingerprint(d: ~base)56 let near = [Fixture.photo("a", at: 0, fingerprint: print), Fixture.photo("b", at: 100, fingerprint: print)]57 let nearGroups = try await CleanupAnalyzer().analyze(near).groups(in: .similar)58 #expect(Fixture.groupIDs(nearGroups) == [["a", "b"]])5960 let far = [Fixture.photo("a", at: 0, fingerprint: print), Fixture.photo("b", at: 200, fingerprint: print)]61 let farAnalysis = try await CleanupAnalyzer().analyze(far)62 #expect(farAnalysis.groups(in: .similar).isEmpty)6364 // A photo in between keeps the session open.65 let bridged = far + [Fixture.photo("m", at: 100, fingerprint: other)]66 let bridgedGroups = try await CleanupAnalyzer().analyze(bridged).groups(in: .similar)67 #expect(Fixture.groupIDs(bridgedGroups) == [["a", "b"]])6869 var wide = CleanupConfiguration()70 wide.similarTimeWindow = 60071 let wideGroups = try await CleanupAnalyzer(configuration: wide).analyze(far).groups(in: .similar)72 #expect(Fixture.groupIDs(wideGroups) == [["a", "b"]])73 }7475 @Test("Lookback closes a group after too many later photos in the same session")76 func lookback() async throws {77 var config = CleanupConfiguration()78 config.similarLookback = 579 var items = [Fixture.photo("a", at: 0, fingerprint: Fixture.fingerprint(d: base))]80 for index in 0..<10 {81 items.append(Fixture.photo("f\(index)", at: Double(index + 1) * 10, fingerprint: Fixture.fingerprint(d: Fixture.flipping(base, bits: 32, seed: UInt64(index)))))82 }83 items.append(Fixture.photo("z", at: 120, fingerprint: Fixture.fingerprint(d: base)))84 let closed = try await CleanupAnalyzer(configuration: config).analyze(items)85 #expect(closed.groups(in: .similar).isEmpty)86 let open = try await CleanupAnalyzer().analyze(items)87 #expect(Fixture.groupIDs(open.groups(in: .similar)) == [["a", "z"]])88 }8990 @Test("Global pass: off by default, strict thresholds across sessions")91 func globalPass() async throws {92 let near = Fixture.flipping(base, bits: 5, seed: 1)93 let items = [94 Fixture.photo("a", at: 0, fingerprint: Fixture.fingerprint(d: base)),95 Fixture.photo("b", at: 50_000, fingerprint: Fixture.fingerprint(d: base)),96 Fixture.photo("c", at: 100_000, fingerprint: Fixture.fingerprint(d: near)),97 Fixture.photo("d", at: 100_030, fingerprint: Fixture.fingerprint(d: Fixture.flipping(near, bits: 2, seed: 2))),98 Fixture.photo("e", at: 150_000, fingerprint: Fixture.fingerprint(d: Fixture.flipping(base, bits: 2, seed: 3))),99 ]100 let off = try await CleanupAnalyzer().analyze(items)101 #expect(Fixture.groupIDs(off.groups(in: .similar)) == [["c", "d"]])102103 var config = CleanupConfiguration()104 config.globalPassEnabled = true105 let on = try await CleanupAnalyzer(configuration: config).analyze(items)106 // a, b (distance 0) and e (distance 2) meet across sessions; c is 5 bits away: too far for the strict pass.107 #expect(Fixture.groupIDs(on.groups(in: .similar)) == [["a", "b", "e"], ["c", "d"]])108109 config.globalDHashThreshold = 6110 config.globalPHashThreshold = 6111 let loose = try await CleanupAnalyzer(configuration: config).analyze(items)112 let merged = loose.groups(in: .similar).contains { group in group.items.contains { $0.id == "a" } && group.items.contains { $0.id == "c" } }113 #expect(merged)114 }115116 @Test("Colour and luminance reject same-structure photos")117 func colourAndLuma() async throws {118 var red = Fixture.plainSignature119 var blue = Fixture.plainSignature120 for cell in 0..<16 {121 red[cell * 3] = 220122 blue[cell * 3 + 2] = 220123 }124 let colours = [125 Fixture.photo("r", at: 0, fingerprint: Fixture.fingerprint(d: base, signature: red)),126 Fixture.photo("b", at: 10, fingerprint: Fixture.fingerprint(d: base, signature: blue)),127 ]128 let colourAnalysis = try await CleanupAnalyzer().analyze(colours)129 #expect(colourAnalysis.groups(in: .similar).isEmpty)130 let lumas = [131 Fixture.photo("dark", at: 0, fingerprint: Fixture.fingerprint(d: base, luma: 30)),132 Fixture.photo("light", at: 10, fingerprint: Fixture.fingerprint(d: base, luma: 200)),133 ]134 let lumaAnalysis = try await CleanupAnalyzer().analyze(lumas)135 #expect(lumaAnalysis.groups(in: .similar).isEmpty)136 }137138 @Test("Videos and screenshots stay out of visual similarity by default")139 func exclusions() async throws {140 let print = Fixture.fingerprint(d: base)141 var video = Fixture.video("v", at: 5, duration: 10)142 video.fingerprint = print143 let items = [144 Fixture.photo("a", at: 0, fingerprint: print), video,145 Fixture.photo("s1", at: 10, fingerprint: print, screenshot: true),146 Fixture.photo("s2", at: 20, fingerprint: print, screenshot: true),147 ]148 let analysis = try await CleanupAnalyzer().analyze(items)149 #expect(analysis.groups(in: .similar).isEmpty)150 var config = CleanupConfiguration()151 config.includeScreenshotsInSimilarity = true152 let included = try await CleanupAnalyzer(configuration: config).analyze(items)153 #expect(Fixture.groupIDs(included.groups(in: .similar)) == [["a", "s1", "s2"]])154 }155156 // MARK: Keeper157158 struct KeeperCase: Sendable, CustomTestStringConvertible {159 var name: String160 var items: [MediaItem]161 var keeper: String162 var rule: KeeperRule163 var testDescription: String { name }164 }165166 static let keeperCases: [KeeperCase] = [167 KeeperCase(name: "favourite beats edited, resolution, size and date", items: [168 Fixture.photo("a", at: 0, width: 4000, height: 3000, size: 900, edited: true),169 Fixture.photo("b", at: 9, width: 100, height: 100, size: 1, favorite: true),170 ], keeper: "b", rule: .favorite),171 KeeperCase(name: "edited beats resolution", items: [172 Fixture.photo("a", at: 0, width: 4000, height: 3000),173 Fixture.photo("b", at: 9, width: 2000, height: 1500, edited: true),174 ], keeper: "b", rule: .edited),175 KeeperCase(name: "highest resolution", items: [176 Fixture.photo("a", at: 0, width: 2000, height: 1500, size: 999),177 Fixture.photo("b", at: 9, width: 4000, height: 3000, size: 1),178 ], keeper: "b", rule: .highestResolution),179 KeeperCase(name: "largest measured size", items: [180 Fixture.photo("a", at: 0, size: 100),181 Fixture.photo("b", at: 9, size: 300),182 Fixture.photo("c", at: 5, size: 200),183 ], keeper: "b", rule: .largestMeasuredSize),184 KeeperCase(name: "size is ignored when one size is unknown; earliest wins", items: [185 Fixture.photo("a", at: 5, size: 100),186 Fixture.photo("b", at: 9, size: 300),187 Fixture.photo("c", at: 1),188 ], keeper: "c", rule: .earliest),189 KeeperCase(name: "two favourites: the rule that separates them is reported", items: [190 Fixture.photo("a", at: 0, width: 100, height: 100, favorite: true),191 Fixture.photo("b", at: 9, width: 200, height: 200, favorite: true),192 Fixture.photo("c", at: 0, width: 900, height: 900),193 ], keeper: "b", rule: .highestResolution),194 KeeperCase(name: "full tie falls back to identifier order", items: [195 Fixture.photo("b", at: 0), Fixture.photo("a", at: 0),196 ], keeper: "a", rule: .identifierOrder),197 KeeperCase(name: "undated items lose to dated ones", items: [198 Fixture.photo("a", at: nil), Fixture.photo("b", at: 50),199 ], keeper: "b", rule: .earliest),200 ]201202 @Test("Keeper rules", arguments: keeperCases)203 func keeperRules(_ testCase: KeeperCase) {204 let forward = CleanupAnalyzer.chooseKeeper(among: testCase.items)205 #expect(forward?.keeper.id == testCase.keeper)206 #expect(forward?.rule == testCase.rule)207 let reversed = CleanupAnalyzer.chooseKeeper(among: Array(testCase.items.reversed()))208 #expect(reversed?.keeper.id == testCase.keeper)209 #expect(reversed?.rule == testCase.rule)210 }211212 @Test("Keeper of an empty list is nil; every rule has an explanation")213 func keeperEdges() {214 #expect(CleanupAnalyzer.chooseKeeper(among: []) == nil)215 for rule in KeeperRule.allCases { #expect(!rule.explanation.isEmpty) }216 }217218 // MARK: Duplicates219220 @Test("Exact duplicates by content hash: measured, non-keepers pre-selected")221 func exactDuplicates() async throws {222 let items = [223 Fixture.photo("a", at: 0, size: 1000, hash: "h1"),224 Fixture.photo("b", at: 5000, size: 1000, hash: "h1", favorite: true),225 Fixture.photo("c", at: 9000, size: 1000, hash: "h1"),226 Fixture.photo("d", at: 10, size: 500, hash: "h2"),227 Fixture.photo("e", at: 20, size: 700, hash: nil),228 ]229 let analysis = try await CleanupAnalyzer().analyze(items)230 let groups = analysis.groups(in: .exactDuplicates)231 #expect(groups.count == 1)232 let group = try #require(groups.first)233 #expect(group.confidence == .measured)234 #expect(group.keeperID == "b")235 #expect(group.keeperRule == .favorite)236 #expect(group.preselectedIDs.sorted() == ["a", "c"])237 #expect(group.candidateIDs.sorted() == ["a", "c"])238 #expect(group.recoverable == .measured(2000))239 #expect(group.measuredPortion == 2000)240 #expect(analysis.preselectedIDs == ["a", "c"])241 #expect(analysis.preselectedTotals == CleanupTotals(candidateCount: 2, measuredBytes: 2000, measuredCount: 2, notMeasuredCount: 0))242 #expect(analysis.groups(in: .probableDuplicates).isEmpty)243 #expect(analysis.recentlyDeletedNote.contains("Recently Deleted"))244 }245246 @Test("Probable duplicates: heuristic, separate category, nothing pre-selected")247 func probableDuplicates() async throws {248 let print = Fixture.fingerprint(d: base, p: ~base)249 let items = [250 Fixture.photo("a", at: 100.2, fingerprint: print),251 Fixture.photo("b", at: 100.7, fingerprint: print),252 Fixture.photo("c", at: 101.1, fingerprint: print), // next second253 Fixture.photo("d", at: 100.4, fingerprint: print, width: 1000, height: 800), // other dimensions254 ]255 let analysis = try await CleanupAnalyzer().analyze(items)256 let groups = analysis.groups(in: .probableDuplicates)257 #expect(Fixture.groupIDs(groups) == [["a", "b"]])258 #expect(groups.first?.confidence == .heuristic)259 #expect(groups.first?.preselectedIDs.isEmpty == true)260 #expect(analysis.preselectedIDs.isEmpty)261 #expect(analysis.groups(in: .exactDuplicates).isEmpty)262 // The keeper of the probable pair still takes part in visual similarity with c and d; the other copy does not.263 let similar = analysis.groups(in: .similar) + analysis.groups(in: .burstLike)264 #expect(Fixture.groupIDs(similar) == [["a", "c", "d"]])265 }266267 @Test("Hashed items with different content are not probable duplicates")268 func hashedAreNotProbable() async throws {269 let print = Fixture.fingerprint(d: base)270 let items = [271 Fixture.photo("a", at: 100.2, fingerprint: print, hash: "x"),272 Fixture.photo("b", at: 100.7, fingerprint: print, hash: "y"),273 ]274 let analysis = try await CleanupAnalyzer().analyze(items)275 #expect(analysis.groups(in: .probableDuplicates).isEmpty)276 #expect(Fixture.groupIDs(analysis.groups(in: .similar)) == [["a", "b"]])277 }278279 @Test("Recoverable bytes: unknown sizes make the amount unavailable, with the measured portion kept")280 func recoverableArithmetic() async throws {281 let items = [282 Fixture.photo("a", at: 0, size: 5000, hash: "h", favorite: true),283 Fixture.photo("b", at: 1, size: 3000, hash: "h"),284 Fixture.photo("c", at: 2, size: nil, hash: "h"),285 Fixture.photo("d", at: 3, size: 1500, hash: "h"),286 ]287 let analysis = try await CleanupAnalyzer().analyze(items)288 let group = try #require(analysis.groups(in: .exactDuplicates).first)289 #expect(group.recoverable.value == nil)290 #expect(group.recoverable.unavailability?.kind == .notRequested)291 #expect(group.measuredPortion == 4500)292 #expect(group.unmeasuredCount == 1)293 let totals = try #require(analysis.category(.exactDuplicates)?.totals)294 #expect(totals == CleanupTotals(candidateCount: 3, measuredBytes: 4500, measuredCount: 2, notMeasuredCount: 1))295 #expect(totals.recoverable.value == nil)296 #expect(analysis.totals == totals)297 // The keeper's size never counts.298 let sizes = CleanupAnalyzer.recoverable(of: [])299 #expect(sizes.reading == .measured(0))300 }301302 // MARK: Bursts303304 @Test("Real bursts, burst-like captures and plain similar groups are told apart")305 func bursts() async throws {306 let one = Fixture.fingerprint(d: base)307 let two = Fixture.fingerprint(d: Fixture.flipping(base, bits: 30, seed: 5))308 let three = Fixture.fingerprint(d: Fixture.flipping(base, bits: 30, seed: 6))309 let four = Fixture.fingerprint(d: Fixture.flipping(base, bits: 30, seed: 7))310 let items = [311 // camera burst312 Fixture.photo("r1", at: 0, fingerprint: one, burst: "B1"),313 Fixture.photo("r2", at: 0.1, fingerprint: one, burst: "B1"),314 Fixture.photo("r3", at: 0.2, fingerprint: one, favorite: true, burst: "B1"),315 // burst-like: four similar photos within two seconds of each other316 Fixture.photo("l1", at: 1000, fingerprint: two, width: 100, height: 100),317 Fixture.photo("l2", at: 1001.5, fingerprint: two, width: 100, height: 100),318 Fixture.photo("l3", at: 1003, fingerprint: two, width: 100, height: 100),319 Fixture.photo("l4", at: 1004.9, fingerprint: two, width: 100, height: 100),320 // similar, but too slow to be burst-like321 Fixture.photo("s1", at: 2000, fingerprint: three),322 Fixture.photo("s2", at: 2030, fingerprint: three),323 Fixture.photo("s3", at: 2060, fingerprint: three),324 // fast, but only two325 Fixture.photo("t1", at: 3000, fingerprint: four),326 Fixture.photo("t2", at: 3001, fingerprint: four),327 // a lone burst member is not a burst328 Fixture.photo("x", at: 4000, fingerprint: one, burst: "B2"),329 ]330 let analysis = try await CleanupAnalyzer().analyze(items)331 let real = analysis.groups(in: .bursts)332 #expect(Fixture.groupIDs(real) == [["r1", "r2", "r3"]])333 #expect(real.first?.id == "bursts:B1")334 #expect(real.first?.keeperID == "r3")335 #expect(real.first?.confidence == Confidence.measured)336 #expect(real.first?.preselectedIDs.isEmpty == true)337 #expect(Fixture.groupIDs(analysis.groups(in: .burstLike)) == [["l1", "l2", "l3", "l4"]])338 #expect(analysis.groups(in: .burstLike).first?.confidence == Confidence.heuristic)339 #expect(Fixture.groupIDs(analysis.groups(in: .similar)) == [["s1", "s2", "s3"], ["t1", "t2"]])340 }341342 // MARK: Lists343344 @Test("Screenshots, old screenshots, videos and Live Photos")345 func lists() async throws {346 var config = CleanupConfiguration()347 config.oldScreenshotAgeDays = 30348 config.referenceDate = Fixture.date(100 * 86_400)349 let items = [350 Fixture.photo("shot-old", at: 10 * 86_400, size: 400, screenshot: true),351 Fixture.photo("shot-new", at: 90 * 86_400, screenshot: true),352 Fixture.video("big", at: 0, duration: 30, size: 900_000_000),353 Fixture.video("bigger", at: 1, duration: 20, size: 2_000_000_000),354 Fixture.video("small", at: 2, duration: 5, size: 3_000_000),355 Fixture.video("unknown-4k-long", at: 3, duration: 600, width: 3840, height: 2160),356 Fixture.video("unknown-hd-2min", at: 4, duration: 130),357 Fixture.video("unknown-short", at: 5, duration: 4),358 Fixture.photo("live", at: 50, size: 5_000_000, kind: .livePhoto),359 ]360 let analysis = try await CleanupAnalyzer(configuration: config).analyze(items)361362 #expect(Fixture.groupIDs(analysis.groups(in: .screenshots)) == [["shot-new", "shot-old"]])363 #expect(Fixture.groupIDs(analysis.groups(in: .oldScreenshots)) == [["shot-old"]])364 let shots = try #require(analysis.groups(in: .screenshots).first)365 #expect(shots.keeperID == nil)366 #expect(shots.preselectedIDs.isEmpty)367 #expect(shots.recoverable.value == nil)368 #expect(shots.measuredPortion == 400)369370 let large = analysis.groups(in: .largeVideos)371 #expect(large.count == 2)372 let measured = try #require(large.first { $0.ordering == .byMeasuredSize })373 #expect(measured.items.map { $0.id } == ["bigger", "big"])374 #expect(measured.recoverable == .measured(2_900_000_000))375 #expect(measured.confidence == .measured)376 #expect(measured.sizeEstimates.isEmpty)377 let estimated = try #require(large.first { $0.ordering == .byEstimate })378 #expect(estimated.items.map { $0.id } == ["unknown-4k-long", "unknown-hd-2min"])379 #expect(estimated.confidence == .heuristic)380 #expect(estimated.sizeEstimates == ["unknown-4k-long": .probablyVeryLarge, "unknown-hd-2min": .probablyLarge])381 // An estimate is never turned into bytes.382 #expect(estimated.recoverable.value == nil)383 #expect(estimated.measuredPortion == 0)384 #expect(estimated.unmeasuredCount == 2)385 for estimate in VideoSizeEstimate.allCases { #expect(estimate.title.contains("estimate")) }386387 #expect(analysis.groups(in: .longVideos).first?.items.map { $0.id } == ["unknown-4k-long", "unknown-hd-2min"])388389 let live = try #require(analysis.groups(in: .livePhotos).first)390 #expect(live.isInformational)391 #expect(live.candidateIDs.isEmpty)392 #expect(analysis.category(.livePhotos)?.totals.candidateCount == 0)393394 // Totals count each candidate once, although shot-old and the long videos sit in two categories.395 #expect(analysis.totals.candidateCount == 6)396 #expect(analysis.totals.measuredBytes == 2_900_000_400)397 #expect(analysis.totals.notMeasuredCount == 3)398 #expect(analysis.preselectedIDs.isEmpty)399 #expect(analysis.statistics.videoCount == 6)400 #expect(analysis.statistics.livePhotoCount == 1)401 #expect(analysis.categories.map { $0.kind } == [.screenshots, .oldScreenshots, .largeVideos, .longVideos, .livePhotos])402 }403404 @Test("Video size estimate buckets")405 func estimates() {406 #expect(VideoSizeEstimate.estimate(duration: 5, pixelWidth: 1920, pixelHeight: 1080) == .probablySmall)407 #expect(VideoSizeEstimate.estimate(duration: 40, pixelWidth: 1920, pixelHeight: 1080) == .probablyMedium)408 #expect(VideoSizeEstimate.estimate(duration: 130, pixelWidth: 1920, pixelHeight: 1080) == .probablyLarge)409 #expect(VideoSizeEstimate.estimate(duration: 600, pixelWidth: 1920, pixelHeight: 1080) == .probablyVeryLarge)410 #expect(VideoSizeEstimate.estimate(duration: -1, pixelWidth: -5, pixelHeight: 10) == .probablySmall)411 }412413 // MARK: Statistics, determinism, cancellation414415 @Test("Statistics: scanned, fingerprinted, skipped with reasons")416 func statistics() async throws {417 let print = Fixture.fingerprint(d: base)418 let items = [419 Fixture.photo("a", at: 0, fingerprint: print, size: 10, hash: "h"),420 Fixture.photo("a", at: 0, fingerprint: print),421 Fixture.photo("nofp", at: 5),422 Fixture.photo("nodate", at: nil, fingerprint: print),423 Fixture.video("v", at: 9, duration: 3),424 ]425 let analysis = try await CleanupAnalyzer().analyze(items)426 let stats = analysis.statistics427 #expect(stats.itemsScanned == 4)428 #expect(stats.itemsFingerprinted == 2)429 #expect(stats.itemsMeasured == 1)430 #expect(stats.itemsHashed == 1)431 #expect(stats.photoCount == 3)432 #expect(stats.skipped == [433 SkippedCount(reason: .duplicateIdentifier, count: 1),434 SkippedCount(reason: .noFingerprint, count: 1),435 SkippedCount(reason: .noCreationDate, count: 1),436 ])437 #expect(stats.duration >= 0)438 let empty = try await CleanupAnalyzer().analyze([])439 #expect(empty.categories.isEmpty)440 #expect(empty.totals == CleanupTotals())441 }442443 @Test("Deterministic: shuffled input gives the same groups")444 func determinism() async throws {445 let items = SyntheticLibrary.items(count: 3000, seed: 11)446 var config = CleanupConfiguration()447 config.globalPassEnabled = true448 config.oldScreenshotAgeDays = 10449 config.referenceDate = Fixture.date(1_000_000)450 let analyzer = CleanupAnalyzer(configuration: config)451 let reference = try await analyzer.analyze(items)452 #expect(!reference.groups(in: .similar).isEmpty)453 #expect(!reference.groups(in: .exactDuplicates).isEmpty)454 var rng = SeededGenerator(seed: 5)455 for _ in 0..<4 {456 let shuffled = items.shuffled(using: &rng)457 let again = try await analyzer.analyze(shuffled)458 #expect(again.categories == reference.categories)459 #expect(again.totals == reference.totals)460 }461 }462463 @Test("Progress is reported and ends with `finished`")464 func progress() async throws {465 let phases = PhaseRecorder()466 _ = try await CleanupAnalyzer().analyze(SyntheticLibrary.items(count: 2000, seed: 3)) { phases.record($0) }467 let seen = phases.phases468 #expect(seen.first == .preparing)469 #expect(seen.last == .finished)470 #expect(seen.contains(.similarity))471 }472473 @Test("Cancellation stops the analysis promptly")474 func cancellation() async throws {475 let items = SyntheticLibrary.items(count: 50_000, seed: 21)476 let clock = ContinuousClock()477 let task = Task { try await CleanupAnalyzer().analyze(items) }478 try await Task.sleep(for: .milliseconds(20))479 let cancelledAt = clock.now480 task.cancel()481 let result = await task.result482 let waited = clock.now - cancelledAt483 switch result {484 case .success: Issue.record("analysis finished before the cancellation was seen")485 case .failure(let error): #expect(error is CancellationError)486 }487 print("CANCELLATION analysis returned \(waited) after cancel")488 #expect(waited < .seconds(2))489 }490491 @Test("Measurement candidates: group members first, then the longest videos, only unmeasured ones")492 func measurementCandidates() async throws {493 let print = Fixture.fingerprint(d: base)494 let items = [495 Fixture.photo("a", at: 0, fingerprint: print),496 Fixture.photo("b", at: 10, fingerprint: print, size: 5, hash: "known"),497 Fixture.photo("lonely", at: 5000, fingerprint: Fixture.fingerprint(d: ~base)),498 Fixture.video("v-long", at: 1, duration: 500),499 Fixture.video("v-mid", at: 2, duration: 50),500 Fixture.video("v-short", at: 3, duration: 5),501 Fixture.video("v-measured", at: 4, duration: 900, size: 77),502 ]503 let analyzer = CleanupAnalyzer()504 let analysis = try await analyzer.analyze(items)505 let candidates = analyzer.measurementCandidates(in: analysis, items: items, topVideosByDuration: 3)506 #expect(candidates == ["a", "v-long", "v-mid"])507 }508509 @Test("Analysis is Codable")510 func codable() async throws {511 let analysis = try await CleanupAnalyzer().analyze(SyntheticLibrary.items(count: 300, seed: 2))512 let data = try JSONEncoder().encode(analysis)513 let decoded = try JSONDecoder().decode(CleanupAnalysis.self, from: data)514 #expect(decoded == analysis)515 }516}517518final class PhaseRecorder: @unchecked Sendable {519 private let lock = NSLock()520 private var storage: [CleanupProgress.Phase] = []521522 func record(_ progress: CleanupProgress) {523 lock.lock()524 if storage.last != progress.phase { storage.append(progress.phase) }525 lock.unlock()526 }527528 var phases: [CleanupProgress.Phase] {529 lock.lock()530 defer { lock.unlock() }531 return storage532 }533}534535/// A synthetic library built directly from fingerprints: sessions of photos, near-duplicate runs, exact duplicates, bursts,536/// screenshots and videos.537enum SyntheticLibrary {538 static func items(count: Int, seed: UInt64) -> [MediaItem] {539 var rng = SeededGenerator(seed: seed)540 var items: [MediaItem] = []541 items.reserveCapacity(count)542 var time: TimeInterval = 0543 var previous: ImageFingerprint?544 for index in 0..<count {545 // Mostly short gaps (one session), sometimes a long pause.546 time += rng.next() % 40 == 0 ? Double(rng.next() % 5000 + 200) : Double(rng.next() % 30) + 0.25547 let id = String(format: "item-%06d", index)548 let roll = rng.next() % 100549 if roll < 4 {550 items.append(Fixture.video(id, at: time, duration: Double(rng.next() % 400), size: roll < 2 ? Int64(rng.next() % 800_000_000) : nil))551 continue552 }553 var fingerprint: ImageFingerprint554 if roll < 24, let last = previous {555 // A near-duplicate of the previous photo: a few bits away.556 fingerprint = last557 fingerprint.dHash = Fixture.flipping(last.dHash, bits: Int(rng.next() % 4), seed: rng.next())558 fingerprint.pHash = Fixture.flipping(last.pHash, bits: Int(rng.next() % 4), seed: rng.next())559 } else {560 var signature = [UInt8](repeating: 0, count: ImageFingerprint.colorSignatureLength)561 for position in signature.indices { signature[position] = UInt8(truncatingIfNeeded: rng.next()) }562 fingerprint = ImageFingerprint(dHash: rng.next(), pHash: rng.next() & ~1, meanLuma: UInt8(truncatingIfNeeded: rng.next()), colorSignature: signature)563 }564 previous = fingerprint565 var item = Fixture.photo(id, at: time, fingerprint: fingerprint, size: roll % 3 == 0 ? nil : Int64(rng.next() % 6_000_000 + 500_000))566 if roll >= 24 && roll < 27, index > 10 {567 // Byte-identical copy of an earlier photo.568 item.contentHash = "hash-\(index - 7)"569 } else if roll % 5 == 0 {570 item.contentHash = "hash-\(index)"571 }572 if roll >= 30 && roll < 36 { item.isScreenshot = true }573 if roll >= 40 && roll < 44 { item.burstIdentifier = "burst-\(index / 6)"; item.isBurstMember = true }574 if roll == 50 { item.kind = .livePhoto }575 if roll == 60 { item.isFavorite = true }576 items.append(item)577 }578 return items579 }580}581