SPB Git

Repository Public License

StoresInspectionTests.swift 24.0 KB 432 lines swift

1// ORIX Convert — Packages/ORIXConvertCore/Tests/ConvertCoreTests/StoresInspectionTests.swift2// Description: History store (SQLite), preset store, FileInspector and SmartDropAnalyzer.3// Author: Simon-Pierre Boucher — contact@spboucher.ai4import Foundation5import SQLite36import Testing7@testable import ConvertCore89private func entry(_ name: String, status: String = "finished", source: FormatID = .png, destination: FormatID = .jpeg, at seconds: Double,10                   error: String? = nil) -> HistoryEntry {11    var e = HistoryEntry(jobID: UUID(), sourcePath: "/Users/me/Pictures/\(name)", destinationFormat: destination, status: status,12                         createdAt: Date(timeIntervalSince1970: seconds - 5), finishedAt: Date(timeIntervalSince1970: seconds))13    e.sourceFormat = source14    e.destinationPaths = status == "finished" ? ["/Users/me/Out/\((name as NSString).deletingPathExtension).\(destination.rawValue)"] : []15    e.error = error16    return e17}1819@Suite struct HistoryStoreTests {20    let temp = TempDir("history")2122    private func store() throws -> ConversionHistoryStore { try ConversionHistoryStore(url: temp.file("db/history.sqlite")) }2324    @Test func roundTripsEveryField() async throws {25        let store = try store()26        var e = entry("cat.png", at: 1_000)27        e.additionalSourcePaths = ["/x/second.png"]28        e.operation = "image.to-pdf"29        e.engine = "image+pdf"30        e.options = ConversionOptions(["quality": .double(0.8), "strip": .bool(true), "width": .int(640), "mode": .string("fit")])31        e.bytesBefore = 1_234_56732        e.bytesAfter = 234_56733        e.duration = 1.2534        e.warnings = [ConversionWarning(.dataLoss, code: "alpha-flattened", "Transparency was flattened.")]35        e.checks = [ValidationCheck("output.format", passed: true, "detected jpeg")]36        e.startedAt = Date(timeIntervalSince1970: 998)37        e.batchID = UUID()38        e.workflowRunID = UUID()39        let id = try await store.record(e)40        #expect(id > 0)41        var expected = e42        expected.id = id43        #expect(try await store.entries() == [expected])44    }4546    @Test func nullableFieldsStayNil() async throws {47        let store = try store()48        var e = HistoryEntry(jobID: UUID(), sourcePath: "/a", destinationFormat: .pdf, status: "failed", createdAt: Date(timeIntervalSince1970: 1), finishedAt: Date(timeIntervalSince1970: 2))49        e.error = "broke"50        try await store.record(e)51        let read = try #require(try await store.entries().first)52        #expect(read.sourceFormat == nil && read.bytesBefore == nil && read.bytesAfter == nil && read.duration == nil && read.startedAt == nil)53        #expect(read.operation == nil && read.batchID == nil && read.error == "broke" && read.destinationPaths.isEmpty)54    }5556    @Test func filtersByStatusFormatAndDate() async throws {57        let store = try store()58        try await store.record(entry("a.png", at: 100))59        try await store.record(entry("b.png", status: "failed", at: 200, error: "Disk full"))60        try await store.record(entry("c.heic", source: .heic, destination: .png, at: 300))61        try await store.record(entry("d.png", status: "cancelled", at: 400))62        var q = HistoryQuery()63        q.statuses = ["failed", "cancelled"]64        #expect(try await store.entries(q).map(\.sourcePath).map { ($0 as NSString).lastPathComponent } == ["d.png", "b.png"])65        q = HistoryQuery()66        q.sourceFormat = .heic67        #expect(try await store.count(q) == 1)68        q = HistoryQuery()69        q.destinationFormat = .jpeg70        #expect(try await store.count(q) == 3)71        q = HistoryQuery()72        q.from = Date(timeIntervalSince1970: 200)73        q.to = Date(timeIntervalSince1970: 400)74        #expect(try await store.entries(q).map(\.finishedAt.timeIntervalSince1970) == [300, 200], "from is inclusive, to exclusive")75        q.statuses = ["finished"]76        #expect(try await store.count(q) == 1)77    }7879    @Test func searchesPathsAndErrorsLiterally() async throws {80        let store = try store()81        try await store.record(entry("holiday_100%.png", at: 1))82        try await store.record(entry("holidayX100Y.png", at: 2))83        try await store.record(entry("other.png", status: "failed", at: 3, error: "Disk full"))84        var q = HistoryQuery()85        q.search = "HOLIDAY"86        #expect(try await store.count(q) == 2, "case-insensitive")87        q.search = "_100%"88        #expect(try await store.count(q) == 1, "LIKE wildcards are escaped")89        q.search = "disk full"90        #expect(try await store.count(q) == 1)91        q.search = "/Out/holiday"92        #expect(try await store.count(q) == 2, "destination paths are searched")93        q.search = "'; DROP TABLE history; --"94        #expect(try await store.count(q) == 0)95        #expect(try await store.count() == 3)96        q.search = "   "97        #expect(try await store.count(q) == 3)98    }99100    @Test func pagesInBothOrders() async throws {101        let store = try store()102        for i in 0..<25 { try await store.record(entry("f\(i).png", at: Double(i))) }103        var q = HistoryQuery()104        q.limit = 10105        #expect(try await store.entries(q).map(\.finishedAt.timeIntervalSince1970) == Array(stride(from: 24.0, through: 15.0, by: -1)))106        q.offset = 20107        #expect(try await store.entries(q).count == 5)108        q.offset = 0109        q.newestFirst = false110        #expect(try await store.entries(q).first?.finishedAt.timeIntervalSince1970 == 0)111        q.limit = -4112        #expect(try await store.entries(q).isEmpty)113        #expect(try await store.count(q) == 25, "count ignores paging")114    }115116    @Test func deletesEntriesAndClearsAll() async throws {117        let store = try store()118        let a = try await store.record(entry("a.png", at: 1))119        let b = try await store.record(entry("b.png", at: 2))120        try await store.record(entry("c.png", at: 3))121        #expect(try await store.delete(ids: [a, b, 9_999]) == 2)122        #expect(try await store.count() == 1)123        try await store.clearAll()124        #expect(try await store.count() == 0)125    }126127    @Test func persistsAcrossReopenAndUsesWAL() async throws {128        do {129            let store = try store()130            try await store.record(entry("kept.png", at: 1))131        }132        let reopened = try store()133        #expect(try await reopened.count() == 1)134        var db: OpaquePointer?135        #expect(sqlite3_open_v2(temp.file("db/history.sqlite").path, &db, SQLITE_OPEN_READONLY, nil) == SQLITE_OK)136        defer { sqlite3_close(db) }137        var statement: OpaquePointer?138        #expect(sqlite3_prepare_v2(db, "PRAGMA journal_mode", -1, &statement, nil) == SQLITE_OK)139        #expect(sqlite3_step(statement) == SQLITE_ROW)140        #expect(String(cString: sqlite3_column_text(statement, 0)) == "wal")141        sqlite3_finalize(statement)142    }143144    @Test func neverStoresFileContents() async throws {145        let bed = TestBed()146        let secret = "TOP-SECRET-CONTENT-\(UUID().uuidString)"147        _ = await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("s.txt", secret)], destinationFormat: .markdown, destination: bed.destination())))148        #expect(try await bed.history.count() == 1)149        for name in ["history.sqlite", "history.sqlite-wal"] where bed.temp.exists(name) {150            let raw = try Data(contentsOf: bed.temp.file(name))151            #expect(raw.range(of: Data(secret.utf8)) == nil && raw.range(of: Data(secret.uppercased().utf8)) == nil)152        }153    }154155    @Test func unopenableLocationThrows() {156        #expect(throws: HistoryError.self) { try ConversionHistoryStore(url: URL(fileURLWithPath: "/dev/null/history.sqlite")) }157    }158159    @Test func concurrentWritersAllLand() async throws {160        let store = try store()161        await withTaskGroup(of: Void.self) { group in162            for i in 0..<60 { group.addTask { _ = try? await store.record(entry("f\(i).png", at: Double(i))) } }163        }164        #expect(try await store.count() == 60)165    }166167    @Test func workflowRunsAreStoredSeparately() async throws {168        let store = try store()169        let workflow = UUID()170        func run(_ seconds: Double, _ id: UUID = workflow) -> WorkflowRunRecord {171            WorkflowRunRecord(id: UUID(), workflowID: id, workflowName: "Web images", startedAt: Date(timeIntervalSince1970: seconds),172                              finishedAt: Date(timeIntervalSince1970: seconds + 2), itemCount: 3, succeeded: 1, failed: 1, filtered: 1, cancelled: 0,173                              items: [.init(sourcePath: "/a.png", status: "succeeded", outputPaths: ["/o/a.jpg"], error: nil),174                                      .init(sourcePath: "/b.png", status: "failed", outputPaths: [], error: "bad")])175        }176        let first = run(10), second = run(20), other = run(30, UUID())177        for record in [first, second, other] { try await store.recordWorkflowRun(record) }178        #expect(try await store.workflowRuns() == [other, second, first])179        #expect(try await store.workflowRuns(workflowID: workflow) == [second, first])180        #expect(try await store.workflowRuns(limit: 1, offset: 1) == [second])181        #expect(try await store.deleteWorkflowRuns(ids: [first.id]) == 1)182        try await store.clearAll()183        #expect(try await store.workflowRuns().isEmpty)184    }185}186187@Suite struct PresetStoreTests {188    let temp = TempDir("presets")189    private var url: URL { temp.file("support/presets.json") }190    private let web = ConversionPreset(name: "Web JPEG", summary: "Small", source: .category(.image), destinationFormat: .jpeg,191                                       options: ConversionOptions(["quality": .double(0.7)]))192193    @Test func startsEmptyWithNoBuiltIns() async {194        let store = PresetStore(fileURL: url)195        let all = await store.all, builtIns = await store.builtInPresets, loadError = await store.loadError196        #expect(all.isEmpty && builtIns.isEmpty && loadError == nil)197    }198199    @Test func addsUpdatesDeletesAndPersists() async throws {200        let store = PresetStore(fileURL: url)201        let added = try await store.add(web)202        var edited = added203        edited.name = "Web JPEG 70"204        edited.operation = "image.resize"205        try await store.update(edited)206        let reopened = PresetStore(fileURL: url)207        #expect(await reopened.userPresets == [edited])208        try await reopened.delete(id: edited.id)209        #expect(await PresetStore(fileURL: url).userPresets.isEmpty)210        await #expect(throws: PresetError.notFound) { try await reopened.delete(id: edited.id) }211        await #expect(throws: PresetError.notFound) { try await reopened.update(edited) }212    }213214    @Test func builtInsAreRegisteredNotStoredAndReadOnly() async throws {215        let store = PresetStore(fileURL: url)216        let builtIn = ConversionPreset(name: "Archive PDF", source: .format(.pdf), destinationFormat: .pdf, operation: "pdf.compress")217        await store.registerBuiltIns([builtIn])218        await store.registerBuiltIns([builtIn])219        let registered = await store.builtInPresets220        #expect(registered.count == 1 && registered[0].isBuiltIn)221        try await store.add(web)222        #expect(await store.all.map(\.name) == ["Archive PDF", "Web JPEG"])223        await #expect(throws: PresetError.builtInIsReadOnly) { try await store.delete(id: builtIn.id) }224        await #expect(throws: PresetError.builtInIsReadOnly) { try await store.update(builtIn) }225        #expect(await PresetStore(fileURL: url).all.map(\.name) == ["Web JPEG"], "built-ins are not written to the user's file")226        let copy = try await store.duplicate(id: builtIn.id)227        #expect(!copy.isBuiltIn && copy.name == "Archive PDF copy" && copy.id != builtIn.id)228    }229230    @Test func scopesMatchFormats() async throws {231        let store = PresetStore(fileURL: url)232        try await store.add(web)233        try await store.add(ConversionPreset(name: "Any to ZIP", source: .any, destinationFormat: .zip))234        try await store.add(ConversionPreset(name: "PNG only", source: .format(.png), destinationFormat: .webp))235        #expect(await store.presets(for: .png).map(\.name) == ["Web JPEG", "Any to ZIP", "PNG only"])236        #expect(await store.presets(for: .heic).map(\.name) == ["Web JPEG", "Any to ZIP"])237        #expect(await store.presets(for: .mp3).map(\.name) == ["Any to ZIP"])238    }239240    @Test func exportsAndImportsWithNewIdentifiers() async throws {241        let store = PresetStore(fileURL: url)242        let a = try await store.add(web)243        try await store.add(ConversionPreset(name: "Other", source: .any, destinationFormat: .png))244        let all = try await store.exportData()245        let one = try await store.exportData(ids: [a.id])246        let target = PresetStore(fileURL: temp.file("other/presets.json"))247        #expect(try await target.importData(one).map(\.name) == ["Web JPEG"])248        let imported = try await target.importData(all)249        #expect(imported.count == 2 && !imported.map(\.id).contains(a.id))250        #expect(await target.userPresets.count == 3)251        #expect(await target.userPresets[0].options.double("quality") == 0.7)252        await #expect(throws: PresetError.self) { try await target.importData(Data("not json".utf8)) }253        #expect(await target.userPresets.count == 3, "a bad import changes nothing")254    }255256    @Test func idCollisionsGetAFreshIdentifier() async throws {257        let store = PresetStore(fileURL: url)258        let first = try await store.add(web)259        let second = try await store.add(first)260        let count = await store.userPresets.count261        #expect(first.id != second.id && count == 2)262    }263264    @Test func corruptFileIsReportedAndNotOverwrittenSilently() async throws {265        try FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true)266        try Data("{ broken".utf8).write(to: url)267        let store = PresetStore(fileURL: url)268        let loadError = await store.loadError, user = await store.userPresets269        #expect(loadError != nil && user.isEmpty)270        #expect(try String(contentsOf: url, encoding: .utf8) == "{ broken")271    }272}273274@Suite struct FileInspectorTests {275    @Test func mergesEngineInspectionWithFileFacts() async throws {276        let temp = TempDir("inspect")277        let url = temp.write("notes.txt", "one\ntwo\nthree\n")278        try FileManager.default.setAttributes([.posixPermissions: 0o640], ofItemAtPath: url.path)279        let file = await FileInspector(registry: FormatRegistry(engines: [TextEngine(), HTMLEngine()])).inspect(url)280        #expect(file.detection.format == .txt && file.engineID == "mock-text" && file.engineError == nil)281        #expect(file.inspection.rowCount == 3, "typed values come from the engine")282        #expect(file.facts.size == 14 && file.facts.posixPermissions == 0o640 && file.facts.modified != nil && !file.facts.isDirectory)283        #expect(file.facts.uti == "public.plain-text")284        let fields = Dictionary(uniqueKeysWithValues: file.inspection.fields.map { ($0.id, $0.value) })285        #expect(fields["File/Size"] == "14 bytes" && fields["File/Permissions"] == "640" && fields["File/Format"] == "Plain Text")286        #expect(fields["File/Text encoding"] == "ascii" && fields["Text/Characters"] == "14")287        #expect(fields["File/Quarantine attribute"] == "Absent")288    }289290    @Test func reportsQuarantineAndWhereFromAttributes() async throws {291        let temp = TempDir("xattr")292        let url = temp.write("download.txt", "x")293        #expect(setxattr(url.path, "com.apple.quarantine", "0081;00000000;Test;", 19, 0, 0) == 0)294        #expect(setxattr(url.path, "com.apple.metadata:kMDItemWhereFroms", "x", 1, 0, 0) == 0)295        let facts = FileInspector.facts(of: url)296        #expect(facts.isQuarantined && facts.hasWhereFroms)297        #expect(!FileInspector.facts(of: temp.write("plain.txt", "x")).isQuarantined)298    }299300    @Test func worksWithoutAnEngineAndForUnknownFormats() async {301        let temp = TempDir("inspect")302        let registry = FormatRegistry(engines: [TextEngine(), HTMLEngine()])303        let png = await FileInspector(registry: registry).inspect(temp.write("a.png", bytes: Fixtures.png))304        #expect(png.detection.format == .png && png.engineID == nil && png.inspection.pixelWidth == nil, "unknown stays nil")305        #expect(png.inspection.fields.contains { $0.id == "File/Size" })306        let blob = await FileInspector(registry: registry).inspect(temp.write("b.xyz", bytes: [1, 2, 3, 0, 9]))307        #expect(blob.detection.format == nil && blob.inspection.format == "unknown")308        let shallow = await FileInspector(registry: registry).inspect(temp.write("c.txt", "x"), deep: false)309        #expect(shallow.engineID == nil && shallow.inspection.rowCount == nil)310    }311312    @Test func engineFailureIsReportedNotThrown() async {313        let temp = TempDir("inspect")314        let url = temp.write("latin.txt", bytes: ascii("caf") + [0xE9] + ascii(" au lait, s'il vous plait\n"))315        let file = await FileInspector(registry: FormatRegistry(engines: [TextEngine(), HTMLEngine()])).inspect(url)316        #expect(file.detection.format == .txt && file.engineID == nil && file.engineError != nil)317        #expect(file.facts.size == 30)318    }319320    @Test func factsOfMissingAndDirectoryItems() {321        let temp = TempDir("facts")322        #expect(FileInspector.facts(of: temp.file("ghost")).size == nil)323        let dir = FileInspector.facts(of: temp.subdirectory("d"))324        #expect(dir.isDirectory && dir.size == nil)325        #expect(FileInspector.facts(of: temp.write(".hidden", "x")).isHidden)326    }327}328329@Suite struct SmartDropTests {330    let temp = TempDir("drop")331    private var registry: FormatRegistry {332        FormatRegistry(engines: [TextEngine(), HTMLEngine(), StubEngine(id: "arch", paths: [ConversionPath(.folder, .zip, engine: "arch", operation: "archive.create", fidelity: .lossless),333                                                                               ConversionPath(.png, .jpeg, engine: "arch", fidelity: .lossy)])])334    }335336    @Test func analysesAMixedBatchAndGroupsByFormat() async {337        let urls = [temp.write("a.txt", "one"), temp.write("b.txt", "two"), temp.write("c.png", bytes: Fixtures.png), temp.write("d.xyz", bytes: [1, 2, 0, 3])]338        let analysis = await SmartDropAnalyzer(registry: registry).analyze(urls)339        #expect(analysis.items.map(\.url.lastPathComponent) == ["a.txt", "b.txt", "c.png", "d.xyz"], "input order is kept")340        #expect(analysis.isMixed && analysis.groups.map(\.id) == ["txt", "png", "unknown"])341        #expect(analysis.groups[0].urls.count == 2 && analysis.groups[0].recommended.first?.destination == .markdown)342        let text = analysis.items[0]343        #expect(text.isSupported && text.targets.contains { $0.destination == .yaml } && text.operations.contains { $0.operation == "text.merge" })344        #expect(text.chainedDestinations == [.html, .jpeg] && text.inspection.rowCount == 1 && text.warnings.isEmpty)345        #expect(analysis.items[2].targets.map(\.destination) == [.jpeg])346        #expect(analysis.items[3].warnings.map(\.code) == ["unknown-format"] && !analysis.items[3].isSupported)347        #expect(analysis.commonDestinations.isEmpty, "txt and png share no destination")348        #expect(analysis.skipped.isEmpty && !analysis.truncated)349    }350351    @Test func commonDestinationsOfAHomogeneousBatch() async {352        let analysis = await SmartDropAnalyzer(registry: registry).analyze([temp.write("a.txt", "one"), temp.write("b.txt", "two")])353        #expect(!analysis.isMixed && analysis.commonDestinations.contains(.markdown) && analysis.commonDestinations.contains(.yaml))354    }355356    @Test func flagsMismatchEmptyAndUnsupportedFiles() async {357        let urls = [temp.write("fake.png", "this is text"), temp.write("empty.txt", ""), temp.write("song.flac", bytes: ascii("fLaC") + zeros(40))]358        let analysis = await SmartDropAnalyzer(registry: registry).analyze(urls)359        #expect(analysis.items[0].format == .txt && analysis.items[0].warnings.map(\.code) == ["extension-mismatch"])360        #expect(analysis.items[1].warnings.map(\.code).contains("empty-file"))361        #expect(analysis.items[2].warnings.map(\.code) == ["unsupported-format"])362    }363364    @Test func reportsMissingComponents() async {365        var engine = TextEngine()366        engine.available = false367        let analysis = await SmartDropAnalyzer(registry: FormatRegistry(engines: [engine])).analyze([temp.write("a.txt", "x")])368        #expect(analysis.items[0].warnings.map(\.code) == ["engine-unavailable"])369        #expect(analysis.items[0].warnings[0].message.contains("mock component missing"))370    }371372    @Test func expandsFoldersSkippingHiddenFilesAndLinks() async throws {373        temp.write("album/one.txt", "1")374        temp.write("album/.DS_Store", "x")375        temp.write("album/sub/two.txt", "2")376        temp.write("album/sub/.secret/three.txt", "3")377        temp.write("album/Letter.pages/Index.zip", "PK")378        try FileManager.default.createSymbolicLink(at: temp.file("album/loop"), withDestinationURL: temp.file("album"))379        let analysis = await SmartDropAnalyzer(registry: registry).analyze([temp.file("album"), temp.write("top.txt", "t"), temp.file("ghost.txt")])380        #expect(analysis.items.map(\.url.lastPathComponent) == ["Letter.pages", "one.txt", "two.txt", "top.txt"])381        #expect(analysis.items[0].format == .pages, "an iWork package is one item, not a folder to expand")382        #expect(analysis.folders.count == 1 && analysis.folders[0].fileCount == 3)383        #expect(analysis.folders[0].targets.map(\.operation) == ["archive.create"], "the folder itself can be archived")384        #expect(analysis.skipped.map(\.url.lastPathComponent) == ["ghost.txt"])385    }386387    @Test func boundsTheNumberOfFiles() async {388        for i in 0..<30 { temp.write("many/f\(i).txt", "x") }389        let analysis = await SmartDropAnalyzer(registry: registry, configuration: .init(maxFiles: 10, deepInspection: false)).analyze([temp.file("many")])390        #expect(analysis.items.count == 10 && analysis.truncated)391        #expect(analysis.items.allSatisfy { $0.inspection.rowCount == nil }, "deep inspection was off")392    }393394    @Test func duplicatesAreAnalysedOnce() async {395        let url = temp.write("a.txt", "x")396        let analysis = await SmartDropAnalyzer(registry: registry).analyze([url, url, temp.url])397        #expect(analysis.items.count == 1)398    }399400    @Test func inspectionConcurrencyIsBounded() async {401        struct SlowInspector: ConversionEngine {402            let id = "slow-inspect"403            let displayName = "Slow"404            let probe: ConcurrencyProbe405            var paths: [ConversionPath] { [ConversionPath(.txt, .markdown, engine: id, fidelity: .lossless)] }406            func inspect(_ url: URL, format: FormatID) async throws -> FileInspection {407                probe.enter(url.lastPathComponent)408                defer { probe.leave() }409                try await Task.sleep(nanoseconds: 30_000_000)410                return FileInspection(format: format)411            }412            func convert(_ request: ConversionRequest, context: ConversionContext) async throws -> ConversionOutput { ConversionOutput(files: []) }413        }414        let probe = ConcurrencyProbe()415        let urls = (0..<12).map { temp.write("c\($0).txt", "x") }416        let analyzer = SmartDropAnalyzer(registry: FormatRegistry(engines: [SlowInspector(probe: probe)]), configuration: .init(maxConcurrentInspections: 3))417        let analysis = await analyzer.analyze(urls)418        #expect(analysis.items.count == 12 && probe.maximum == 3)419    }420421    @Test func preflightListsLimitationsChainsAndEngineWarnings() async throws {422        let analyzer = SmartDropAnalyzer(registry: registry)423        let item = try #require(await analyzer.analyze([temp.write("a.txt", "x")]).items.first)424        let direct = try analyzer.preflight(item, to: .markdown, options: ConversionOptions(["warn": .bool(true)]))425        #expect(!direct.plan.isChained)426        #expect(direct.warnings.map(\.code) == ["limitation", "preflight-test"] && direct.warnings[0].message == "Text is upper-cased.")427        let chained = try analyzer.preflight(item, to: .html)428        #expect(chained.plan.isChained && chained.warnings.map(\.code) == ["chained-plan", "limitation", "limitation"])429        #expect(throws: ConversionError.self) { try analyzer.preflight(item, to: .mp3) }430    }431}432
Files main
Menu

Theme