SPB Git

Repository Public License

WatcherTests.swift 24.0 KB 451 lines swift

1// ORIX Convert — Packages/ORIXConvertCore/Tests/ConvertCoreTests/WatcherTests.swift2// Description: FolderWatcher with real temporary folders: arrival, stability, temporary names, loop prevention,3//              duplicates, pause/resume, persistence, rules by format/category/glob, presets and workflows.4// Author: Simon-Pierre Boucher — contact@spboucher.ai5import Foundation6import Testing7@testable import ConvertCore89/// A watcher over the mock engine with short timings.10private struct WatchBed {11    let bed = TestBed()12    let presets: PresetStore13    let workflows: WorkflowLibrary14    let watcher: FolderWatcher15    let configuration = WatcherConfiguration(settleInterval: 0.3, tickInterval: 0.05, rescanInterval: 0.2)1617    init() {18        presets = PresetStore(fileURL: bed.temp.file("support/presets.json"))19        workflows = WorkflowLibrary(fileURL: bed.temp.file("support/workflows.json"))20        watcher = WatchBed.makeWatcher(bed, presets, workflows, configuration)21    }2223    static func makeWatcher(_ bed: TestBed, _ presets: PresetStore, _ workflows: WorkflowLibrary, _ configuration: WatcherConfiguration) -> FolderWatcher {24        FolderWatcher(runner: WorkflowRunner(queue: bed.queue, history: bed.history, workDirectories: bed.workDirectories), presets: presets, workflows: workflows,25                      storeURL: bed.temp.file("support/watch.json"), configuration: configuration)26    }2728    /// A second watcher over the same stored state (as after relaunching the app).29    func relaunched() -> FolderWatcher { WatchBed.makeWatcher(bed, presets, workflows, configuration) }3031    var inbox: URL { bed.temp.subdirectory("inbox") }32    var outbox: URL { bed.temp.subdirectory("outbox") }3334    func rule(_ name: String = "Text to Markdown", match: WatchRule.Match = .formats([.txt]), to format: FormatID = .markdown, folder: URL? = nil, destination: URL? = nil) -> WatchRule {35        WatchRule(name: name, folder: folder ?? inbox, match: match, action: .convert(destination: format, operation: nil, options: .init()),36                  destinationFolder: destination ?? outbox)37    }3839    @discardableResult40    func drop(_ name: String, _ text: String = "hello\n", into folder: String = "inbox") -> URL { bed.temp.write(folder + "/" + name, text) }4142    func activity(_ kind: WatchActivity.Kind) async -> [WatchActivity] { await watcher.activity.filter { $0.kind == kind } }4344    /// Waits long enough for a file that should NOT be processed to have been processed if the watcher were wrong.45    func settle() async { try? await Task.sleep(nanoseconds: 1_200_000_000) }46}4748@Suite struct FolderWatcherTests {49    @Test func processesAFileThatArrives() async throws {50        let w = WatchBed()51        try await w.watcher.addRule(w.rule())52        await w.watcher.start()53        w.drop("note.txt", "arrived\n")54        #expect(await eventually { w.bed.temp.exists("outbox/note.md") })55        #expect(w.bed.temp.read("outbox/note.md") == "ARRIVED\n" && w.bed.temp.exists("inbox/note.txt"))56        #expect(await eventually { await w.activity(.processed).count == 1 })57        let activity = try #require(await w.activity(.processed).first)58        #expect(activity.path.hasSuffix("/inbox/note.txt") && activity.outputs.count == 1 && activity.outputs[0].hasSuffix("/outbox/note.md"))59        await w.watcher.stop()60        #expect(await !w.watcher.isRunning)61    }6263    @Test func filesAlreadyPresentAreLeftAloneByDefault() async throws {64        let w = WatchBed()65        w.drop("old.txt")66        try await w.watcher.addRule(w.rule())67        await w.watcher.start()68        w.drop("new.txt")69        #expect(await eventually { w.bed.temp.exists("outbox/new.md") })70        await w.settle()71        #expect(w.bed.temp.list("outbox") == ["new.md"])72        await w.watcher.stop()73    }7475    @Test func existingFilesAreProcessedWhenTheRuleAsks() async throws {76        let w = WatchBed()77        w.drop("old.txt")78        var rule = w.rule()79        rule.processExistingFiles = true80        try await w.watcher.addRule(rule)81        await w.watcher.start()82        #expect(await eventually { w.bed.temp.exists("outbox/old.md") })83        await w.watcher.stop()84    }8586    @Test func waitsUntilTheFileStopsChanging() async throws {87        let w = WatchBed()88        try await w.watcher.addRule(w.rule())89        await w.watcher.start()90        let url = w.drop("growing.txt", "part 0\n")91        let handle = try FileHandle(forWritingTo: url)92        for chunk in 1...8 {93            try await Task.sleep(nanoseconds: 120_000_000)   // shorter than the settle interval94            try handle.seekToEnd()95            try handle.write(contentsOf: Data("part \(chunk)\n".utf8))96            #expect(!w.bed.temp.exists("outbox/growing.md"), "not processed while still being written")97        }98        try handle.close()99        #expect(await eventually { w.bed.temp.exists("outbox/growing.md") })100        #expect(w.bed.temp.read("outbox/growing.md").split(separator: "\n").count == 9, "the complete file was converted")101        await w.settle()102        #expect(await w.activity(.processed).count == 1, "processed exactly once")103        await w.watcher.stop()104    }105106    @Test func ignoresPartialDownloadsUntilTheyGetTheirFinalName() async throws {107        let w = WatchBed()108        try await w.watcher.addRule(w.rule(match: .any))109        await w.watcher.start()110        let partial = w.drop("report.txt.crdownload", "downloading\n")111        w.drop("movie.txt.part")112        w.drop(".hidden.txt")113        w.drop("~$lock.txt")114        w.drop("Unconfirmed.download")115        await w.settle()116        #expect(w.bed.temp.list("outbox").isEmpty)117        #expect(await w.watcher.activity.isEmpty)118        try FileManager.default.moveItem(at: partial, to: w.inbox.appendingPathComponent("report.txt"))119        #expect(await eventually { w.bed.temp.exists("outbox/report.md") })120        await w.watcher.stop()121    }122123    @Test func temporaryNames() {124        for name in [".DS_Store", ".hidden", "a.crdownload", "b.PART", "c.partial", "d.download", "e.tmp", "f.temp", "~$doc.docx", "g.txt~", "Thumbs.db", "h.opdownload", "i.swp"] {125            #expect(FolderWatcher.isTemporaryName(name), "\(name)")126        }127        for name in ["photo.jpg", "part.txt", "download.pdf", "my.temporary.notes.txt", "tmp"] {128            #expect(!FolderWatcher.isTemporaryName(name), "\(name)")129        }130    }131132    @Test func emptyFilesAndFoldersAreNotProcessed() async throws {133        let w = WatchBed()134        try await w.watcher.addRule(w.rule(match: .any))135        await w.watcher.start()136        w.drop("empty.txt", "")137        _ = w.bed.temp.subdirectory("inbox/Folder.txt")138        w.drop("sub/inside.txt")139        await w.settle()140        #expect(w.bed.temp.list("outbox").isEmpty, "empty file, directory and (non-recursive rule) nested file")141        await w.watcher.stop()142    }143144    @Test func subfoldersAreWatchedWhenAsked() async throws {145        let w = WatchBed()146        var rule = w.rule()147        rule.includeSubfolders = true148        try await w.watcher.addRule(rule)149        await w.watcher.start()150        w.drop("deep/er/inside.txt")151        #expect(await eventually { w.bed.temp.exists("outbox/inside.md") })152        await w.watcher.stop()153    }154155    @Test func matchesByDetectedFormatCategoryAndGlob() async throws {156        let w = WatchBed()157        let byFormat = w.bed.temp.subdirectory("by-format"), byCategory = w.bed.temp.subdirectory("by-category"), byGlob = w.bed.temp.subdirectory("by-glob")158        try await w.watcher.addRule(w.rule("format", match: .formats([.txt]), folder: byFormat, destination: w.bed.temp.subdirectory("out-format")))159        try await w.watcher.addRule(w.rule("category", match: .categories([.document]), to: .yaml, folder: byCategory, destination: w.bed.temp.subdirectory("out-category")))160        try await w.watcher.addRule(w.rule("glob", match: .glob("scan-*.txt"), folder: byGlob, destination: w.bed.temp.subdirectory("out-glob")))161        await w.watcher.start()162        w.drop("text-without-extension", "words only\n", into: "by-format")      // detected as txt from content163        w.bed.temp.write("by-format/picture.txt", bytes: Fixtures.png)            // named .txt, really a PNG164        w.drop("plain.txt", into: "by-category")165        w.bed.temp.write("by-category/image.png", bytes: Fixtures.png)166        w.drop("scan-001.txt", into: "by-glob")167        w.drop("other.txt", into: "by-glob")168        #expect(await eventually { w.bed.temp.exists("out-format/text-without-extension.md") && w.bed.temp.exists("out-category/plain.yaml") && w.bed.temp.exists("out-glob/scan-001.md") })169        await w.settle()170        #expect(w.bed.temp.list("out-format") == ["text-without-extension.md"] && w.bed.temp.list("out-category") == ["plain.yaml"] && w.bed.temp.list("out-glob") == ["scan-001.md"])171        await w.watcher.stop()172    }173174    @Test func refusesRulesThatWouldProcessTheirOwnResults() async throws {175        let w = WatchBed()176        let inbox = w.inbox177        func risk(_ rule: WatchRule) async -> Bool {178            do { try await w.watcher.addRule(rule); return false } catch WatchError.loopRisk { return true } catch { Issue.record("unexpected \(error)"); return false }179        }180        #expect(await risk(w.rule(match: .any, destination: inbox)), "any file, same folder")181        #expect(await risk(w.rule(match: .glob("*"), destination: inbox)))182        #expect(await risk(w.rule(match: .glob("*.md"), destination: inbox)), "the outputs are .md files")183        #expect(await risk(w.rule(match: .formats([.txt, .markdown]), destination: inbox)))184        #expect(await risk(w.rule(match: .categories([.document]), destination: inbox)), "markdown is a document too")185        var recursive = w.rule(match: .any, destination: inbox.appendingPathComponent("Converted"))186        recursive.includeSubfolders = true187        #expect(await risk(recursive), "a destination inside a recursively watched folder")188        #expect(await w.watcher.rules.isEmpty)189        // Safe variants of the same idea.190        #expect(await !risk(w.rule("formats", match: .formats([.txt]), destination: inbox)), "outputs are markdown, the rule takes only txt")191        #expect(await !risk(w.rule("glob", match: .glob("*.txt"), destination: inbox)))192        #expect(await !risk(w.rule("category", match: .categories([.image]), destination: inbox)))193        #expect(await !risk(w.rule("subfolder", match: .any, destination: inbox.appendingPathComponent("Converted"))), "not recursive: the subfolder is not watched")194        #expect(await w.watcher.rules.count == 4)195    }196197    @Test func convertingInPlaceDoesNotLoop() async throws {198        let w = WatchBed()199        try await w.watcher.addRule(w.rule(match: .formats([.txt]), destination: w.inbox))200        await w.watcher.start()201        w.drop("a.txt", "one\n")202        #expect(await eventually { w.bed.temp.exists("inbox/a.md") })203        await w.settle()204        #expect(w.bed.temp.list("inbox") == ["a.md", "a.txt"])205        #expect(await w.activity(.processed).count == 1)206        await w.watcher.stop()207    }208209    @Test func filesProducedByTheWatcherAreNeverProcessedByAnotherRule() async throws {210        let w = WatchBed()211        let middle = w.bed.temp.subdirectory("middle")212        try await w.watcher.addRule(w.rule("first", match: .formats([.txt]), to: .markdown, destination: middle))213        try await w.watcher.addRule(w.rule("second", match: .formats([.markdown]), to: .html, folder: middle, destination: w.outbox))214        await w.watcher.start()215        w.drop("chain.txt", "x\n")216        #expect(await eventually { w.bed.temp.exists("middle/chain.md") })217        await w.settle()218        #expect(w.bed.temp.list("outbox").isEmpty, "the watcher's own output is not picked up by the second rule")219        // A file the user puts there is processed normally.220        w.drop("mine.md", "# Mine\n\n**my** own [file](x)\n", into: "middle")221        #expect(await eventually { w.bed.temp.exists("outbox/mine.html") })222        await w.watcher.stop()223    }224225    @Test func unchangedFilesAreNotProcessedTwiceAndChangedOnesAre() async throws {226        let w = WatchBed()227        try await w.watcher.addRule(w.rule())228        await w.watcher.start()229        w.drop("a.txt", "version 1\n")230        #expect(await eventually { w.bed.temp.exists("outbox/a.md") })231        await w.watcher.scanNow()232        await w.settle()233        #expect(w.bed.temp.list("outbox") == ["a.md"], "rescans do not reprocess an unchanged file")234        w.drop("a.txt", "version 2 is longer\n")235        #expect(await eventually { w.bed.temp.exists("outbox/a 2.md") })236        #expect(w.bed.temp.read("outbox/a 2.md") == "VERSION 2 IS LONGER\n")237        #expect(await w.activity(.duplicate).isEmpty)238        await w.watcher.stop()239    }240241    @Test func identicalContentDroppedAgainIsReportedAsDuplicate() async throws {242        let w = WatchBed()243        try await w.watcher.addRule(w.rule())244        await w.watcher.start()245        w.drop("first.txt", "same bytes\n")246        #expect(await eventually { w.bed.temp.exists("outbox/first.md") })247        w.drop("copy of first.txt", "same bytes\n")248        #expect(await eventually { await w.activity(.duplicate).count == 1 })249        #expect(await w.activity(.duplicate).first?.path.hasSuffix("copy of first.txt") == true)250        // Same path removed and dropped again with the same content.251        try FileManager.default.removeItem(at: w.inbox.appendingPathComponent("first.txt"))252        try await Task.sleep(nanoseconds: 1_100_000_000)253        w.drop("first.txt", "same bytes\n")254        #expect(await eventually { await w.activity(.duplicate).count == 2 })255        #expect(w.bed.temp.list("outbox") == ["first.md"])256        await w.watcher.stop()257    }258259    @Test func duplicateContentCheckCanBeTurnedOff() async throws {260        let bed = TestBed()261        let watcher = FolderWatcher(runner: WorkflowRunner(queue: bed.queue, workDirectories: bed.workDirectories), presets: PresetStore(fileURL: bed.temp.file("p.json")),262                                    workflows: WorkflowLibrary(fileURL: bed.temp.file("w.json")), storeURL: bed.temp.file("watch.json"),263                                    configuration: WatcherConfiguration(settleInterval: 0.2, tickInterval: 0.05, rescanInterval: 0.2, skipDuplicateContent: false))264        let inbox = bed.temp.subdirectory("inbox"), outbox = bed.temp.subdirectory("outbox")265        try await watcher.addRule(WatchRule(name: "r", folder: inbox, match: .formats([.txt]), action: .convert(destination: .markdown, operation: nil, options: .init()), destinationFolder: outbox))266        await watcher.start()267        bed.temp.write("inbox/a.txt", "same\n")268        bed.temp.write("inbox/b.txt", "same\n")269        #expect(await eventually { bed.temp.list("outbox") == ["a.md", "b.md"] })270        await watcher.stop()271    }272273    @Test func pausesAndResumesARule() async throws {274        let w = WatchBed()275        let rule = try await w.watcher.addRule(w.rule())276        await w.watcher.start()277        try await w.watcher.setRulePaused(rule.id, true)278        #expect(await w.watcher.rule(id: rule.id)?.isPaused == true)279        w.drop("while-paused.txt")280        await w.settle()281        #expect(w.bed.temp.list("outbox").isEmpty)282        try await w.watcher.setRulePaused(rule.id, false)283        w.drop("after-resume.txt", "different\n")284        #expect(await eventually { w.bed.temp.exists("outbox/after-resume.md") })285        await w.settle()286        #expect(w.bed.temp.list("outbox") == ["after-resume.md"], "what arrived during the pause is not processed afterwards")287        await w.watcher.stop()288    }289290    @Test func pausesAndResumesEverything() async throws {291        let w = WatchBed()292        try await w.watcher.addRule(w.rule())293        await w.watcher.start()294        await w.watcher.pauseAll()295        #expect(await w.watcher.isPaused)296        w.drop("held.txt")297        await w.settle()298        #expect(w.bed.temp.list("outbox").isEmpty)299        await w.watcher.resumeAll()300        w.drop("released.txt", "other\n")301        #expect(await eventually { w.bed.temp.exists("outbox/released.md") })302        #expect(await w.watcher.rules.first?.isPaused == false, "the rule's own flag is untouched")303        await w.watcher.stop()304    }305306    @Test func nothingHappensBeforeStartOrAfterStop() async throws {307        let w = WatchBed()308        try await w.watcher.addRule(w.rule())309        w.drop("before.txt")310        await w.settle()311        #expect(w.bed.temp.list("outbox").isEmpty)312        await w.watcher.start()313        await w.watcher.stop()314        w.drop("after.txt", "z\n")315        await w.settle()316        #expect(w.bed.temp.list("outbox").isEmpty)317    }318319    @Test func rulesActivityAndFingerprintsPersist() async throws {320        let w = WatchBed()321        let rule = try await w.watcher.addRule(w.rule())322        await w.watcher.start()323        w.drop("a.txt", "persist me\n")324        #expect(await eventually { await w.activity(.processed).count == 1 })325        await w.watcher.stop()326327        let again = w.relaunched()328        let restored = try #require(await again.rules.first)329        #expect(restored.id == rule.id && restored.name == rule.name && restored.folderBookmark != nil && restored.destinationBookmark != nil)330        #expect(await again.activity.count == 1)331        var existing = restored332        existing.processExistingFiles = true333        try await again.updateRule(existing)334        await again.start()335        w.drop("b.txt", "new after relaunch\n")336        #expect(await eventually { w.bed.temp.exists("outbox/b.md") })337        await w.settle()338        #expect(w.bed.temp.list("outbox") == ["a.md", "b.md"], "a.txt was remembered and not converted again")339        try await again.clearActivity()340        #expect(await again.activity.isEmpty)341        await again.stop()342    }343344    @Test func rulesFollowARenamedFolderThroughTheirBookmark() async throws {345        let w = WatchBed()346        try await w.watcher.addRule(w.rule())347        let moved = w.bed.temp.file("inbox-renamed")348        try FileManager.default.moveItem(at: w.bed.temp.file("inbox"), to: moved)349        let again = w.relaunched()350        await again.start()351        #expect(await again.rules.first?.folder.lastPathComponent == "inbox-renamed")352        w.drop("x.txt", "moved\n", into: "inbox-renamed")353        #expect(await eventually { w.bed.temp.exists("outbox/x.md") })354        await again.stop()355    }356357    @Test func rulesCanRunPresetsAndWorkflows() async throws {358        let w = WatchBed()359        let preset = try await w.presets.add(ConversionPreset(name: "Loud", source: .format(.txt), destinationFormat: .markdown, options: ConversionOptions(["suffix": .string("!!")])))360        let special = w.bed.temp.subdirectory("special")361        let workflow = Workflow(name: "Sorted", nodes: [362            .branch(condition: .nameMatches("special*"), then: [.convert(destination: .yaml, operation: nil, options: .init()), .export(folder: special, conflictPolicy: .rename)], otherwise: []),363            .convert(destination: .markdown, operation: nil, options: .init()),364        ])365        try await w.workflows.save(workflow)366        let presetInbox = w.bed.temp.subdirectory("preset-inbox"), flowInbox = w.bed.temp.subdirectory("flow-inbox")367        try await w.watcher.addRule(WatchRule(name: "preset", folder: presetInbox, match: .formats([.txt]), action: .preset(preset.id), destinationFolder: w.bed.temp.subdirectory("preset-out")))368        try await w.watcher.addRule(WatchRule(name: "flow", folder: flowInbox, match: .formats([.txt]), action: .workflow(workflow.id), destinationFolder: w.bed.temp.subdirectory("flow-out")))369        await w.watcher.start()370        w.drop("p.txt", "quiet\n", into: "preset-inbox")371        w.drop("special-1.txt", "s\n", into: "flow-inbox")372        w.drop("plain.txt", "p\n", into: "flow-inbox")373        #expect(await eventually { w.bed.temp.read("preset-out/p.md") == "QUIET\n!!" })374        // The workflow's own export node decides for the special file; the path without export uses the rule's folder.375        #expect(await eventually { w.bed.temp.exists("special/special-1.yaml") && w.bed.temp.exists("flow-out/plain.md") })376        await w.settle()377        #expect(w.bed.temp.list("flow-out") == ["plain.md"] && w.bed.temp.list("special") == ["special-1.yaml"])378        await w.watcher.stop()379    }380381    @Test func invalidRulesAreRefused() async throws {382        let w = WatchBed()383        await #expect(throws: WatchError.self) { try await w.watcher.addRule(w.rule(folder: w.bed.temp.file("does-not-exist"))) }384        await #expect(throws: WatchError.self) {385            try await w.watcher.addRule(WatchRule(name: "ghost preset", folder: w.inbox, match: .any, action: .preset(UUID()), destinationFolder: w.outbox))386        }387        await #expect(throws: WatchError.self) {388            try await w.watcher.addRule(WatchRule(name: "ghost flow", folder: w.inbox, match: .any, action: .workflow(UUID()), destinationFolder: w.outbox))389        }390        await #expect(throws: WatchError.self) { try await w.watcher.removeRule(id: UUID()) }391        await #expect(throws: WatchError.self) { try await w.watcher.updateRule(w.rule()) }392        await #expect(throws: WatchError.self) { try await w.watcher.setRulePaused(UUID(), true) }393        #expect(await w.watcher.rules.isEmpty)394        let created = try await w.watcher.addRule(w.rule(destination: w.bed.temp.file("made/on/demand")))395        #expect(w.bed.temp.exists("made/on/demand"), "the destination folder is created")396        try await w.watcher.removeRule(id: created.id)397        #expect(await w.watcher.rules.isEmpty)398    }399400    @Test func failuresAreRecordedAndDoNotStopTheWatcher() async throws {401        let w = WatchBed()402        try await w.watcher.addRule(w.rule("fails", to: .rtf))403        await w.watcher.start()404        w.drop("bad.txt", "one\n")405        #expect(await eventually { await w.activity(.failed).count == 1 })406        #expect(await w.activity(.failed).first?.detail.contains("mock failure") == true)407        w.drop("bad2.txt", "two\n")408        #expect(await eventually { await w.activity(.failed).count == 2 })409        await w.settle()410        #expect(await w.watcher.activity.count == 2, "a failed file is not retried in a loop")411        await w.watcher.stop()412    }413414    @Test func eventsAnnounceActivityAndState() async throws {415        let w = WatchBed()416        let stream = await w.watcher.events()417        let collector = Task { () -> (sawRules: Bool, sawRunning: Bool, activity: WatchActivity?) in418            var sawRules = false, sawRunning = false419            for await event in stream {420                switch event {421                case .rulesChanged(let rules): sawRules = rules.count == 1422                case .stateChanged(let running, _): if running { sawRunning = true }423                case .activity(let activity): return (sawRules, sawRunning, activity)424                }425            }426            return (sawRules, sawRunning, nil)427        }428        try await w.watcher.addRule(w.rule())429        await w.watcher.start()430        w.drop("e.txt")431        let result = await collector.value432        #expect(result.sawRules && result.sawRunning && result.activity?.kind == .processed)433        await w.watcher.stop()434    }435436    @Test func activityLogIsBounded() async throws {437        let bed = TestBed()438        let watcher = FolderWatcher(runner: WorkflowRunner(queue: bed.queue, workDirectories: bed.workDirectories), presets: PresetStore(fileURL: bed.temp.file("p.json")),439                                    workflows: WorkflowLibrary(fileURL: bed.temp.file("w.json")), storeURL: bed.temp.file("watch.json"),440                                    configuration: WatcherConfiguration(settleInterval: 0.1, tickInterval: 0.03, rescanInterval: 0.1, maxActivityEntries: 10))441        let inbox = bed.temp.subdirectory("inbox")442        try await watcher.addRule(WatchRule(name: "r", folder: inbox, match: .formats([.txt]), action: .convert(destination: .markdown, operation: nil, options: .init()),443                                            destinationFolder: bed.temp.subdirectory("outbox")))444        await watcher.start()445        for i in 0..<16 { bed.temp.write("inbox/f\(i).txt", "content \(i)\n") }446        #expect(await eventually { bed.temp.list("outbox").count == 16 })447        #expect(await eventually { await watcher.activity.count == 10 })448        await watcher.stop()449    }450}451
Files main
Menu

Theme