Repository Public License
1// ORIX Convert — Packages/ORIXConvertCore/Tests/ConvertCoreTests/StoresInspectionTests.swift2// Description: History store (SQLite), preset store, FileInspector and SmartDropAnalyzer.3// Author: Simon-Pierre Boucher — contact@spboucher.ai4import Foundation5import SQLite36import Testing7@testable import ConvertCore89private func entry(_ name: String, status: String = "finished", source: FormatID = .png, destination: FormatID = .jpeg, at seconds: Double,10 error: String? = nil) -> HistoryEntry {11 var e = HistoryEntry(jobID: UUID(), sourcePath: "/Users/me/Pictures/\(name)", destinationFormat: destination, status: status,12 createdAt: Date(timeIntervalSince1970: seconds - 5), finishedAt: Date(timeIntervalSince1970: seconds))13 e.sourceFormat = source14 e.destinationPaths = status == "finished" ? ["/Users/me/Out/\((name as NSString).deletingPathExtension).\(destination.rawValue)"] : []15 e.error = error16 return e17}1819@Suite struct HistoryStoreTests {20 let temp = TempDir("history")2122 private func store() throws -> ConversionHistoryStore { try ConversionHistoryStore(url: temp.file("db/history.sqlite")) }2324 @Test func roundTripsEveryField() async throws {25 let store = try store()26 var e = entry("cat.png", at: 1_000)27 e.additionalSourcePaths = ["/x/second.png"]28 e.operation = "image.to-pdf"29 e.engine = "image+pdf"30 e.options = ConversionOptions(["quality": .double(0.8), "strip": .bool(true), "width": .int(640), "mode": .string("fit")])31 e.bytesBefore = 1_234_56732 e.bytesAfter = 234_56733 e.duration = 1.2534 e.warnings = [ConversionWarning(.dataLoss, code: "alpha-flattened", "Transparency was flattened.")]35 e.checks = [ValidationCheck("output.format", passed: true, "detected jpeg")]36 e.startedAt = Date(timeIntervalSince1970: 998)37 e.batchID = UUID()38 e.workflowRunID = UUID()39 let id = try await store.record(e)40 #expect(id > 0)41 var expected = e42 expected.id = id43 #expect(try await store.entries() == [expected])44 }4546 @Test func nullableFieldsStayNil() async throws {47 let store = try store()48 var e = HistoryEntry(jobID: UUID(), sourcePath: "/a", destinationFormat: .pdf, status: "failed", createdAt: Date(timeIntervalSince1970: 1), finishedAt: Date(timeIntervalSince1970: 2))49 e.error = "broke"50 try await store.record(e)51 let read = try #require(try await store.entries().first)52 #expect(read.sourceFormat == nil && read.bytesBefore == nil && read.bytesAfter == nil && read.duration == nil && read.startedAt == nil)53 #expect(read.operation == nil && read.batchID == nil && read.error == "broke" && read.destinationPaths.isEmpty)54 }5556 @Test func filtersByStatusFormatAndDate() async throws {57 let store = try store()58 try await store.record(entry("a.png", at: 100))59 try await store.record(entry("b.png", status: "failed", at: 200, error: "Disk full"))60 try await store.record(entry("c.heic", source: .heic, destination: .png, at: 300))61 try await store.record(entry("d.png", status: "cancelled", at: 400))62 var q = HistoryQuery()63 q.statuses = ["failed", "cancelled"]64 #expect(try await store.entries(q).map(\.sourcePath).map { ($0 as NSString).lastPathComponent } == ["d.png", "b.png"])65 q = HistoryQuery()66 q.sourceFormat = .heic67 #expect(try await store.count(q) == 1)68 q = HistoryQuery()69 q.destinationFormat = .jpeg70 #expect(try await store.count(q) == 3)71 q = HistoryQuery()72 q.from = Date(timeIntervalSince1970: 200)73 q.to = Date(timeIntervalSince1970: 400)74 #expect(try await store.entries(q).map(\.finishedAt.timeIntervalSince1970) == [300, 200], "from is inclusive, to exclusive")75 q.statuses = ["finished"]76 #expect(try await store.count(q) == 1)77 }7879 @Test func searchesPathsAndErrorsLiterally() async throws {80 let store = try store()81 try await store.record(entry("holiday_100%.png", at: 1))82 try await store.record(entry("holidayX100Y.png", at: 2))83 try await store.record(entry("other.png", status: "failed", at: 3, error: "Disk full"))84 var q = HistoryQuery()85 q.search = "HOLIDAY"86 #expect(try await store.count(q) == 2, "case-insensitive")87 q.search = "_100%"88 #expect(try await store.count(q) == 1, "LIKE wildcards are escaped")89 q.search = "disk full"90 #expect(try await store.count(q) == 1)91 q.search = "/Out/holiday"92 #expect(try await store.count(q) == 2, "destination paths are searched")93 q.search = "'; DROP TABLE history; --"94 #expect(try await store.count(q) == 0)95 #expect(try await store.count() == 3)96 q.search = " "97 #expect(try await store.count(q) == 3)98 }99100 @Test func pagesInBothOrders() async throws {101 let store = try store()102 for i in 0..<25 { try await store.record(entry("f\(i).png", at: Double(i))) }103 var q = HistoryQuery()104 q.limit = 10105 #expect(try await store.entries(q).map(\.finishedAt.timeIntervalSince1970) == Array(stride(from: 24.0, through: 15.0, by: -1)))106 q.offset = 20107 #expect(try await store.entries(q).count == 5)108 q.offset = 0109 q.newestFirst = false110 #expect(try await store.entries(q).first?.finishedAt.timeIntervalSince1970 == 0)111 q.limit = -4112 #expect(try await store.entries(q).isEmpty)113 #expect(try await store.count(q) == 25, "count ignores paging")114 }115116 @Test func deletesEntriesAndClearsAll() async throws {117 let store = try store()118 let a = try await store.record(entry("a.png", at: 1))119 let b = try await store.record(entry("b.png", at: 2))120 try await store.record(entry("c.png", at: 3))121 #expect(try await store.delete(ids: [a, b, 9_999]) == 2)122 #expect(try await store.count() == 1)123 try await store.clearAll()124 #expect(try await store.count() == 0)125 }126127 @Test func persistsAcrossReopenAndUsesWAL() async throws {128 do {129 let store = try store()130 try await store.record(entry("kept.png", at: 1))131 }132 let reopened = try store()133 #expect(try await reopened.count() == 1)134 var db: OpaquePointer?135 #expect(sqlite3_open_v2(temp.file("db/history.sqlite").path, &db, SQLITE_OPEN_READONLY, nil) == SQLITE_OK)136 defer { sqlite3_close(db) }137 var statement: OpaquePointer?138 #expect(sqlite3_prepare_v2(db, "PRAGMA journal_mode", -1, &statement, nil) == SQLITE_OK)139 #expect(sqlite3_step(statement) == SQLITE_ROW)140 #expect(String(cString: sqlite3_column_text(statement, 0)) == "wal")141 sqlite3_finalize(statement)142 }143144 @Test func neverStoresFileContents() async throws {145 let bed = TestBed()146 let secret = "TOP-SECRET-CONTENT-\(UUID().uuidString)"147 _ = await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("s.txt", secret)], destinationFormat: .markdown, destination: bed.destination())))148 #expect(try await bed.history.count() == 1)149 for name in ["history.sqlite", "history.sqlite-wal"] where bed.temp.exists(name) {150 let raw = try Data(contentsOf: bed.temp.file(name))151 #expect(raw.range(of: Data(secret.utf8)) == nil && raw.range(of: Data(secret.uppercased().utf8)) == nil)152 }153 }154155 @Test func unopenableLocationThrows() {156 #expect(throws: HistoryError.self) { try ConversionHistoryStore(url: URL(fileURLWithPath: "/dev/null/history.sqlite")) }157 }158159 @Test func concurrentWritersAllLand() async throws {160 let store = try store()161 await withTaskGroup(of: Void.self) { group in162 for i in 0..<60 { group.addTask { _ = try? await store.record(entry("f\(i).png", at: Double(i))) } }163 }164 #expect(try await store.count() == 60)165 }166167 @Test func workflowRunsAreStoredSeparately() async throws {168 let store = try store()169 let workflow = UUID()170 func run(_ seconds: Double, _ id: UUID = workflow) -> WorkflowRunRecord {171 WorkflowRunRecord(id: UUID(), workflowID: id, workflowName: "Web images", startedAt: Date(timeIntervalSince1970: seconds),172 finishedAt: Date(timeIntervalSince1970: seconds + 2), itemCount: 3, succeeded: 1, failed: 1, filtered: 1, cancelled: 0,173 items: [.init(sourcePath: "/a.png", status: "succeeded", outputPaths: ["/o/a.jpg"], error: nil),174 .init(sourcePath: "/b.png", status: "failed", outputPaths: [], error: "bad")])175 }176 let first = run(10), second = run(20), other = run(30, UUID())177 for record in [first, second, other] { try await store.recordWorkflowRun(record) }178 #expect(try await store.workflowRuns() == [other, second, first])179 #expect(try await store.workflowRuns(workflowID: workflow) == [second, first])180 #expect(try await store.workflowRuns(limit: 1, offset: 1) == [second])181 #expect(try await store.deleteWorkflowRuns(ids: [first.id]) == 1)182 try await store.clearAll()183 #expect(try await store.workflowRuns().isEmpty)184 }185}186187@Suite struct PresetStoreTests {188 let temp = TempDir("presets")189 private var url: URL { temp.file("support/presets.json") }190 private let web = ConversionPreset(name: "Web JPEG", summary: "Small", source: .category(.image), destinationFormat: .jpeg,191 options: ConversionOptions(["quality": .double(0.7)]))192193 @Test func startsEmptyWithNoBuiltIns() async {194 let store = PresetStore(fileURL: url)195 let all = await store.all, builtIns = await store.builtInPresets, loadError = await store.loadError196 #expect(all.isEmpty && builtIns.isEmpty && loadError == nil)197 }198199 @Test func addsUpdatesDeletesAndPersists() async throws {200 let store = PresetStore(fileURL: url)201 let added = try await store.add(web)202 var edited = added203 edited.name = "Web JPEG 70"204 edited.operation = "image.resize"205 try await store.update(edited)206 let reopened = PresetStore(fileURL: url)207 #expect(await reopened.userPresets == [edited])208 try await reopened.delete(id: edited.id)209 #expect(await PresetStore(fileURL: url).userPresets.isEmpty)210 await #expect(throws: PresetError.notFound) { try await reopened.delete(id: edited.id) }211 await #expect(throws: PresetError.notFound) { try await reopened.update(edited) }212 }213214 @Test func builtInsAreRegisteredNotStoredAndReadOnly() async throws {215 let store = PresetStore(fileURL: url)216 let builtIn = ConversionPreset(name: "Archive PDF", source: .format(.pdf), destinationFormat: .pdf, operation: "pdf.compress")217 await store.registerBuiltIns([builtIn])218 await store.registerBuiltIns([builtIn])219 let registered = await store.builtInPresets220 #expect(registered.count == 1 && registered[0].isBuiltIn)221 try await store.add(web)222 #expect(await store.all.map(\.name) == ["Archive PDF", "Web JPEG"])223 await #expect(throws: PresetError.builtInIsReadOnly) { try await store.delete(id: builtIn.id) }224 await #expect(throws: PresetError.builtInIsReadOnly) { try await store.update(builtIn) }225 #expect(await PresetStore(fileURL: url).all.map(\.name) == ["Web JPEG"], "built-ins are not written to the user's file")226 let copy = try await store.duplicate(id: builtIn.id)227 #expect(!copy.isBuiltIn && copy.name == "Archive PDF copy" && copy.id != builtIn.id)228 }229230 @Test func scopesMatchFormats() async throws {231 let store = PresetStore(fileURL: url)232 try await store.add(web)233 try await store.add(ConversionPreset(name: "Any to ZIP", source: .any, destinationFormat: .zip))234 try await store.add(ConversionPreset(name: "PNG only", source: .format(.png), destinationFormat: .webp))235 #expect(await store.presets(for: .png).map(\.name) == ["Web JPEG", "Any to ZIP", "PNG only"])236 #expect(await store.presets(for: .heic).map(\.name) == ["Web JPEG", "Any to ZIP"])237 #expect(await store.presets(for: .mp3).map(\.name) == ["Any to ZIP"])238 }239240 @Test func exportsAndImportsWithNewIdentifiers() async throws {241 let store = PresetStore(fileURL: url)242 let a = try await store.add(web)243 try await store.add(ConversionPreset(name: "Other", source: .any, destinationFormat: .png))244 let all = try await store.exportData()245 let one = try await store.exportData(ids: [a.id])246 let target = PresetStore(fileURL: temp.file("other/presets.json"))247 #expect(try await target.importData(one).map(\.name) == ["Web JPEG"])248 let imported = try await target.importData(all)249 #expect(imported.count == 2 && !imported.map(\.id).contains(a.id))250 #expect(await target.userPresets.count == 3)251 #expect(await target.userPresets[0].options.double("quality") == 0.7)252 await #expect(throws: PresetError.self) { try await target.importData(Data("not json".utf8)) }253 #expect(await target.userPresets.count == 3, "a bad import changes nothing")254 }255256 @Test func idCollisionsGetAFreshIdentifier() async throws {257 let store = PresetStore(fileURL: url)258 let first = try await store.add(web)259 let second = try await store.add(first)260 let count = await store.userPresets.count261 #expect(first.id != second.id && count == 2)262 }263264 @Test func corruptFileIsReportedAndNotOverwrittenSilently() async throws {265 try FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true)266 try Data("{ broken".utf8).write(to: url)267 let store = PresetStore(fileURL: url)268 let loadError = await store.loadError, user = await store.userPresets269 #expect(loadError != nil && user.isEmpty)270 #expect(try String(contentsOf: url, encoding: .utf8) == "{ broken")271 }272}273274@Suite struct FileInspectorTests {275 @Test func mergesEngineInspectionWithFileFacts() async throws {276 let temp = TempDir("inspect")277 let url = temp.write("notes.txt", "one\ntwo\nthree\n")278 try FileManager.default.setAttributes([.posixPermissions: 0o640], ofItemAtPath: url.path)279 let file = await FileInspector(registry: FormatRegistry(engines: [TextEngine(), HTMLEngine()])).inspect(url)280 #expect(file.detection.format == .txt && file.engineID == "mock-text" && file.engineError == nil)281 #expect(file.inspection.rowCount == 3, "typed values come from the engine")282 #expect(file.facts.size == 14 && file.facts.posixPermissions == 0o640 && file.facts.modified != nil && !file.facts.isDirectory)283 #expect(file.facts.uti == "public.plain-text")284 let fields = Dictionary(uniqueKeysWithValues: file.inspection.fields.map { ($0.id, $0.value) })285 #expect(fields["File/Size"] == "14 bytes" && fields["File/Permissions"] == "640" && fields["File/Format"] == "Plain Text")286 #expect(fields["File/Text encoding"] == "ascii" && fields["Text/Characters"] == "14")287 #expect(fields["File/Quarantine attribute"] == "Absent")288 }289290 @Test func reportsQuarantineAndWhereFromAttributes() async throws {291 let temp = TempDir("xattr")292 let url = temp.write("download.txt", "x")293 #expect(setxattr(url.path, "com.apple.quarantine", "0081;00000000;Test;", 19, 0, 0) == 0)294 #expect(setxattr(url.path, "com.apple.metadata:kMDItemWhereFroms", "x", 1, 0, 0) == 0)295 let facts = FileInspector.facts(of: url)296 #expect(facts.isQuarantined && facts.hasWhereFroms)297 #expect(!FileInspector.facts(of: temp.write("plain.txt", "x")).isQuarantined)298 }299300 @Test func worksWithoutAnEngineAndForUnknownFormats() async {301 let temp = TempDir("inspect")302 let registry = FormatRegistry(engines: [TextEngine(), HTMLEngine()])303 let png = await FileInspector(registry: registry).inspect(temp.write("a.png", bytes: Fixtures.png))304 #expect(png.detection.format == .png && png.engineID == nil && png.inspection.pixelWidth == nil, "unknown stays nil")305 #expect(png.inspection.fields.contains { $0.id == "File/Size" })306 let blob = await FileInspector(registry: registry).inspect(temp.write("b.xyz", bytes: [1, 2, 3, 0, 9]))307 #expect(blob.detection.format == nil && blob.inspection.format == "unknown")308 let shallow = await FileInspector(registry: registry).inspect(temp.write("c.txt", "x"), deep: false)309 #expect(shallow.engineID == nil && shallow.inspection.rowCount == nil)310 }311312 @Test func engineFailureIsReportedNotThrown() async {313 let temp = TempDir("inspect")314 let url = temp.write("latin.txt", bytes: ascii("caf") + [0xE9] + ascii(" au lait, s'il vous plait\n"))315 let file = await FileInspector(registry: FormatRegistry(engines: [TextEngine(), HTMLEngine()])).inspect(url)316 #expect(file.detection.format == .txt && file.engineID == nil && file.engineError != nil)317 #expect(file.facts.size == 30)318 }319320 @Test func factsOfMissingAndDirectoryItems() {321 let temp = TempDir("facts")322 #expect(FileInspector.facts(of: temp.file("ghost")).size == nil)323 let dir = FileInspector.facts(of: temp.subdirectory("d"))324 #expect(dir.isDirectory && dir.size == nil)325 #expect(FileInspector.facts(of: temp.write(".hidden", "x")).isHidden)326 }327}328329@Suite struct SmartDropTests {330 let temp = TempDir("drop")331 private var registry: FormatRegistry {332 FormatRegistry(engines: [TextEngine(), HTMLEngine(), StubEngine(id: "arch", paths: [ConversionPath(.folder, .zip, engine: "arch", operation: "archive.create", fidelity: .lossless),333 ConversionPath(.png, .jpeg, engine: "arch", fidelity: .lossy)])])334 }335336 @Test func analysesAMixedBatchAndGroupsByFormat() async {337 let urls = [temp.write("a.txt", "one"), temp.write("b.txt", "two"), temp.write("c.png", bytes: Fixtures.png), temp.write("d.xyz", bytes: [1, 2, 0, 3])]338 let analysis = await SmartDropAnalyzer(registry: registry).analyze(urls)339 #expect(analysis.items.map(\.url.lastPathComponent) == ["a.txt", "b.txt", "c.png", "d.xyz"], "input order is kept")340 #expect(analysis.isMixed && analysis.groups.map(\.id) == ["txt", "png", "unknown"])341 #expect(analysis.groups[0].urls.count == 2 && analysis.groups[0].recommended.first?.destination == .markdown)342 let text = analysis.items[0]343 #expect(text.isSupported && text.targets.contains { $0.destination == .yaml } && text.operations.contains { $0.operation == "text.merge" })344 #expect(text.chainedDestinations == [.html, .jpeg] && text.inspection.rowCount == 1 && text.warnings.isEmpty)345 #expect(analysis.items[2].targets.map(\.destination) == [.jpeg])346 #expect(analysis.items[3].warnings.map(\.code) == ["unknown-format"] && !analysis.items[3].isSupported)347 #expect(analysis.commonDestinations.isEmpty, "txt and png share no destination")348 #expect(analysis.skipped.isEmpty && !analysis.truncated)349 }350351 @Test func commonDestinationsOfAHomogeneousBatch() async {352 let analysis = await SmartDropAnalyzer(registry: registry).analyze([temp.write("a.txt", "one"), temp.write("b.txt", "two")])353 #expect(!analysis.isMixed && analysis.commonDestinations.contains(.markdown) && analysis.commonDestinations.contains(.yaml))354 }355356 @Test func flagsMismatchEmptyAndUnsupportedFiles() async {357 let urls = [temp.write("fake.png", "this is text"), temp.write("empty.txt", ""), temp.write("song.flac", bytes: ascii("fLaC") + zeros(40))]358 let analysis = await SmartDropAnalyzer(registry: registry).analyze(urls)359 #expect(analysis.items[0].format == .txt && analysis.items[0].warnings.map(\.code) == ["extension-mismatch"])360 #expect(analysis.items[1].warnings.map(\.code).contains("empty-file"))361 #expect(analysis.items[2].warnings.map(\.code) == ["unsupported-format"])362 }363364 @Test func reportsMissingComponents() async {365 var engine = TextEngine()366 engine.available = false367 let analysis = await SmartDropAnalyzer(registry: FormatRegistry(engines: [engine])).analyze([temp.write("a.txt", "x")])368 #expect(analysis.items[0].warnings.map(\.code) == ["engine-unavailable"])369 #expect(analysis.items[0].warnings[0].message.contains("mock component missing"))370 }371372 @Test func expandsFoldersSkippingHiddenFilesAndLinks() async throws {373 temp.write("album/one.txt", "1")374 temp.write("album/.DS_Store", "x")375 temp.write("album/sub/two.txt", "2")376 temp.write("album/sub/.secret/three.txt", "3")377 temp.write("album/Letter.pages/Index.zip", "PK")378 try FileManager.default.createSymbolicLink(at: temp.file("album/loop"), withDestinationURL: temp.file("album"))379 let analysis = await SmartDropAnalyzer(registry: registry).analyze([temp.file("album"), temp.write("top.txt", "t"), temp.file("ghost.txt")])380 #expect(analysis.items.map(\.url.lastPathComponent) == ["Letter.pages", "one.txt", "two.txt", "top.txt"])381 #expect(analysis.items[0].format == .pages, "an iWork package is one item, not a folder to expand")382 #expect(analysis.folders.count == 1 && analysis.folders[0].fileCount == 3)383 #expect(analysis.folders[0].targets.map(\.operation) == ["archive.create"], "the folder itself can be archived")384 #expect(analysis.skipped.map(\.url.lastPathComponent) == ["ghost.txt"])385 }386387 @Test func boundsTheNumberOfFiles() async {388 for i in 0..<30 { temp.write("many/f\(i).txt", "x") }389 let analysis = await SmartDropAnalyzer(registry: registry, configuration: .init(maxFiles: 10, deepInspection: false)).analyze([temp.file("many")])390 #expect(analysis.items.count == 10 && analysis.truncated)391 #expect(analysis.items.allSatisfy { $0.inspection.rowCount == nil }, "deep inspection was off")392 }393394 @Test func duplicatesAreAnalysedOnce() async {395 let url = temp.write("a.txt", "x")396 let analysis = await SmartDropAnalyzer(registry: registry).analyze([url, url, temp.url])397 #expect(analysis.items.count == 1)398 }399400 @Test func inspectionConcurrencyIsBounded() async {401 struct SlowInspector: ConversionEngine {402 let id = "slow-inspect"403 let displayName = "Slow"404 let probe: ConcurrencyProbe405 var paths: [ConversionPath] { [ConversionPath(.txt, .markdown, engine: id, fidelity: .lossless)] }406 func inspect(_ url: URL, format: FormatID) async throws -> FileInspection {407 probe.enter(url.lastPathComponent)408 defer { probe.leave() }409 try await Task.sleep(nanoseconds: 30_000_000)410 return FileInspection(format: format)411 }412 func convert(_ request: ConversionRequest, context: ConversionContext) async throws -> ConversionOutput { ConversionOutput(files: []) }413 }414 let probe = ConcurrencyProbe()415 let urls = (0..<12).map { temp.write("c\($0).txt", "x") }416 let analyzer = SmartDropAnalyzer(registry: FormatRegistry(engines: [SlowInspector(probe: probe)]), configuration: .init(maxConcurrentInspections: 3))417 let analysis = await analyzer.analyze(urls)418 #expect(analysis.items.count == 12 && probe.maximum == 3)419 }420421 @Test func preflightListsLimitationsChainsAndEngineWarnings() async throws {422 let analyzer = SmartDropAnalyzer(registry: registry)423 let item = try #require(await analyzer.analyze([temp.write("a.txt", "x")]).items.first)424 let direct = try analyzer.preflight(item, to: .markdown, options: ConversionOptions(["warn": .bool(true)]))425 #expect(!direct.plan.isChained)426 #expect(direct.warnings.map(\.code) == ["limitation", "preflight-test"] && direct.warnings[0].message == "Text is upper-cased.")427 let chained = try analyzer.preflight(item, to: .html)428 #expect(chained.plan.isChained && chained.warnings.map(\.code) == ["chained-plan", "limitation", "limitation"])429 #expect(throws: ConversionError.self) { try analyzer.preflight(item, to: .mp3) }430 }431}432