Repository Public License
1// ORIX Convert — Packages/ORIXConvertCore/Tests/ConvertCoreTests/WatcherTests.swift2// Description: FolderWatcher with real temporary folders: arrival, stability, temporary names, loop prevention,3// duplicates, pause/resume, persistence, rules by format/category/glob, presets and workflows.4// Author: Simon-Pierre Boucher — contact@spboucher.ai5import Foundation6import Testing7@testable import ConvertCore89/// A watcher over the mock engine with short timings.10private struct WatchBed {11 let bed = TestBed()12 let presets: PresetStore13 let workflows: WorkflowLibrary14 let watcher: FolderWatcher15 let configuration = WatcherConfiguration(settleInterval: 0.3, tickInterval: 0.05, rescanInterval: 0.2)1617 init() {18 presets = PresetStore(fileURL: bed.temp.file("support/presets.json"))19 workflows = WorkflowLibrary(fileURL: bed.temp.file("support/workflows.json"))20 watcher = WatchBed.makeWatcher(bed, presets, workflows, configuration)21 }2223 static func makeWatcher(_ bed: TestBed, _ presets: PresetStore, _ workflows: WorkflowLibrary, _ configuration: WatcherConfiguration) -> FolderWatcher {24 FolderWatcher(runner: WorkflowRunner(queue: bed.queue, history: bed.history, workDirectories: bed.workDirectories), presets: presets, workflows: workflows,25 storeURL: bed.temp.file("support/watch.json"), configuration: configuration)26 }2728 /// A second watcher over the same stored state (as after relaunching the app).29 func relaunched() -> FolderWatcher { WatchBed.makeWatcher(bed, presets, workflows, configuration) }3031 var inbox: URL { bed.temp.subdirectory("inbox") }32 var outbox: URL { bed.temp.subdirectory("outbox") }3334 func rule(_ name: String = "Text to Markdown", match: WatchRule.Match = .formats([.txt]), to format: FormatID = .markdown, folder: URL? = nil, destination: URL? = nil) -> WatchRule {35 WatchRule(name: name, folder: folder ?? inbox, match: match, action: .convert(destination: format, operation: nil, options: .init()),36 destinationFolder: destination ?? outbox)37 }3839 @discardableResult40 func drop(_ name: String, _ text: String = "hello\n", into folder: String = "inbox") -> URL { bed.temp.write(folder + "/" + name, text) }4142 func activity(_ kind: WatchActivity.Kind) async -> [WatchActivity] { await watcher.activity.filter { $0.kind == kind } }4344 /// Waits long enough for a file that should NOT be processed to have been processed if the watcher were wrong.45 func settle() async { try? await Task.sleep(nanoseconds: 1_200_000_000) }46}4748@Suite struct FolderWatcherTests {49 @Test func processesAFileThatArrives() async throws {50 let w = WatchBed()51 try await w.watcher.addRule(w.rule())52 await w.watcher.start()53 w.drop("note.txt", "arrived\n")54 #expect(await eventually { w.bed.temp.exists("outbox/note.md") })55 #expect(w.bed.temp.read("outbox/note.md") == "ARRIVED\n" && w.bed.temp.exists("inbox/note.txt"))56 #expect(await eventually { await w.activity(.processed).count == 1 })57 let activity = try #require(await w.activity(.processed).first)58 #expect(activity.path.hasSuffix("/inbox/note.txt") && activity.outputs.count == 1 && activity.outputs[0].hasSuffix("/outbox/note.md"))59 await w.watcher.stop()60 #expect(await !w.watcher.isRunning)61 }6263 @Test func filesAlreadyPresentAreLeftAloneByDefault() async throws {64 let w = WatchBed()65 w.drop("old.txt")66 try await w.watcher.addRule(w.rule())67 await w.watcher.start()68 w.drop("new.txt")69 #expect(await eventually { w.bed.temp.exists("outbox/new.md") })70 await w.settle()71 #expect(w.bed.temp.list("outbox") == ["new.md"])72 await w.watcher.stop()73 }7475 @Test func existingFilesAreProcessedWhenTheRuleAsks() async throws {76 let w = WatchBed()77 w.drop("old.txt")78 var rule = w.rule()79 rule.processExistingFiles = true80 try await w.watcher.addRule(rule)81 await w.watcher.start()82 #expect(await eventually { w.bed.temp.exists("outbox/old.md") })83 await w.watcher.stop()84 }8586 @Test func waitsUntilTheFileStopsChanging() async throws {87 let w = WatchBed()88 try await w.watcher.addRule(w.rule())89 await w.watcher.start()90 let url = w.drop("growing.txt", "part 0\n")91 let handle = try FileHandle(forWritingTo: url)92 for chunk in 1...8 {93 try await Task.sleep(nanoseconds: 120_000_000) // shorter than the settle interval94 try handle.seekToEnd()95 try handle.write(contentsOf: Data("part \(chunk)\n".utf8))96 #expect(!w.bed.temp.exists("outbox/growing.md"), "not processed while still being written")97 }98 try handle.close()99 #expect(await eventually { w.bed.temp.exists("outbox/growing.md") })100 #expect(w.bed.temp.read("outbox/growing.md").split(separator: "\n").count == 9, "the complete file was converted")101 await w.settle()102 #expect(await w.activity(.processed).count == 1, "processed exactly once")103 await w.watcher.stop()104 }105106 @Test func ignoresPartialDownloadsUntilTheyGetTheirFinalName() async throws {107 let w = WatchBed()108 try await w.watcher.addRule(w.rule(match: .any))109 await w.watcher.start()110 let partial = w.drop("report.txt.crdownload", "downloading\n")111 w.drop("movie.txt.part")112 w.drop(".hidden.txt")113 w.drop("~$lock.txt")114 w.drop("Unconfirmed.download")115 await w.settle()116 #expect(w.bed.temp.list("outbox").isEmpty)117 #expect(await w.watcher.activity.isEmpty)118 try FileManager.default.moveItem(at: partial, to: w.inbox.appendingPathComponent("report.txt"))119 #expect(await eventually { w.bed.temp.exists("outbox/report.md") })120 await w.watcher.stop()121 }122123 @Test func temporaryNames() {124 for name in [".DS_Store", ".hidden", "a.crdownload", "b.PART", "c.partial", "d.download", "e.tmp", "f.temp", "~$doc.docx", "g.txt~", "Thumbs.db", "h.opdownload", "i.swp"] {125 #expect(FolderWatcher.isTemporaryName(name), "\(name)")126 }127 for name in ["photo.jpg", "part.txt", "download.pdf", "my.temporary.notes.txt", "tmp"] {128 #expect(!FolderWatcher.isTemporaryName(name), "\(name)")129 }130 }131132 @Test func emptyFilesAndFoldersAreNotProcessed() async throws {133 let w = WatchBed()134 try await w.watcher.addRule(w.rule(match: .any))135 await w.watcher.start()136 w.drop("empty.txt", "")137 _ = w.bed.temp.subdirectory("inbox/Folder.txt")138 w.drop("sub/inside.txt")139 await w.settle()140 #expect(w.bed.temp.list("outbox").isEmpty, "empty file, directory and (non-recursive rule) nested file")141 await w.watcher.stop()142 }143144 @Test func subfoldersAreWatchedWhenAsked() async throws {145 let w = WatchBed()146 var rule = w.rule()147 rule.includeSubfolders = true148 try await w.watcher.addRule(rule)149 await w.watcher.start()150 w.drop("deep/er/inside.txt")151 #expect(await eventually { w.bed.temp.exists("outbox/inside.md") })152 await w.watcher.stop()153 }154155 @Test func matchesByDetectedFormatCategoryAndGlob() async throws {156 let w = WatchBed()157 let byFormat = w.bed.temp.subdirectory("by-format"), byCategory = w.bed.temp.subdirectory("by-category"), byGlob = w.bed.temp.subdirectory("by-glob")158 try await w.watcher.addRule(w.rule("format", match: .formats([.txt]), folder: byFormat, destination: w.bed.temp.subdirectory("out-format")))159 try await w.watcher.addRule(w.rule("category", match: .categories([.document]), to: .yaml, folder: byCategory, destination: w.bed.temp.subdirectory("out-category")))160 try await w.watcher.addRule(w.rule("glob", match: .glob("scan-*.txt"), folder: byGlob, destination: w.bed.temp.subdirectory("out-glob")))161 await w.watcher.start()162 w.drop("text-without-extension", "words only\n", into: "by-format") // detected as txt from content163 w.bed.temp.write("by-format/picture.txt", bytes: Fixtures.png) // named .txt, really a PNG164 w.drop("plain.txt", into: "by-category")165 w.bed.temp.write("by-category/image.png", bytes: Fixtures.png)166 w.drop("scan-001.txt", into: "by-glob")167 w.drop("other.txt", into: "by-glob")168 #expect(await eventually { w.bed.temp.exists("out-format/text-without-extension.md") && w.bed.temp.exists("out-category/plain.yaml") && w.bed.temp.exists("out-glob/scan-001.md") })169 await w.settle()170 #expect(w.bed.temp.list("out-format") == ["text-without-extension.md"] && w.bed.temp.list("out-category") == ["plain.yaml"] && w.bed.temp.list("out-glob") == ["scan-001.md"])171 await w.watcher.stop()172 }173174 @Test func refusesRulesThatWouldProcessTheirOwnResults() async throws {175 let w = WatchBed()176 let inbox = w.inbox177 func risk(_ rule: WatchRule) async -> Bool {178 do { try await w.watcher.addRule(rule); return false } catch WatchError.loopRisk { return true } catch { Issue.record("unexpected \(error)"); return false }179 }180 #expect(await risk(w.rule(match: .any, destination: inbox)), "any file, same folder")181 #expect(await risk(w.rule(match: .glob("*"), destination: inbox)))182 #expect(await risk(w.rule(match: .glob("*.md"), destination: inbox)), "the outputs are .md files")183 #expect(await risk(w.rule(match: .formats([.txt, .markdown]), destination: inbox)))184 #expect(await risk(w.rule(match: .categories([.document]), destination: inbox)), "markdown is a document too")185 var recursive = w.rule(match: .any, destination: inbox.appendingPathComponent("Converted"))186 recursive.includeSubfolders = true187 #expect(await risk(recursive), "a destination inside a recursively watched folder")188 #expect(await w.watcher.rules.isEmpty)189 // Safe variants of the same idea.190 #expect(await !risk(w.rule("formats", match: .formats([.txt]), destination: inbox)), "outputs are markdown, the rule takes only txt")191 #expect(await !risk(w.rule("glob", match: .glob("*.txt"), destination: inbox)))192 #expect(await !risk(w.rule("category", match: .categories([.image]), destination: inbox)))193 #expect(await !risk(w.rule("subfolder", match: .any, destination: inbox.appendingPathComponent("Converted"))), "not recursive: the subfolder is not watched")194 #expect(await w.watcher.rules.count == 4)195 }196197 @Test func convertingInPlaceDoesNotLoop() async throws {198 let w = WatchBed()199 try await w.watcher.addRule(w.rule(match: .formats([.txt]), destination: w.inbox))200 await w.watcher.start()201 w.drop("a.txt", "one\n")202 #expect(await eventually { w.bed.temp.exists("inbox/a.md") })203 await w.settle()204 #expect(w.bed.temp.list("inbox") == ["a.md", "a.txt"])205 #expect(await w.activity(.processed).count == 1)206 await w.watcher.stop()207 }208209 @Test func filesProducedByTheWatcherAreNeverProcessedByAnotherRule() async throws {210 let w = WatchBed()211 let middle = w.bed.temp.subdirectory("middle")212 try await w.watcher.addRule(w.rule("first", match: .formats([.txt]), to: .markdown, destination: middle))213 try await w.watcher.addRule(w.rule("second", match: .formats([.markdown]), to: .html, folder: middle, destination: w.outbox))214 await w.watcher.start()215 w.drop("chain.txt", "x\n")216 #expect(await eventually { w.bed.temp.exists("middle/chain.md") })217 await w.settle()218 #expect(w.bed.temp.list("outbox").isEmpty, "the watcher's own output is not picked up by the second rule")219 // A file the user puts there is processed normally.220 w.drop("mine.md", "# Mine\n\n**my** own [file](x)\n", into: "middle")221 #expect(await eventually { w.bed.temp.exists("outbox/mine.html") })222 await w.watcher.stop()223 }224225 @Test func unchangedFilesAreNotProcessedTwiceAndChangedOnesAre() async throws {226 let w = WatchBed()227 try await w.watcher.addRule(w.rule())228 await w.watcher.start()229 w.drop("a.txt", "version 1\n")230 #expect(await eventually { w.bed.temp.exists("outbox/a.md") })231 await w.watcher.scanNow()232 await w.settle()233 #expect(w.bed.temp.list("outbox") == ["a.md"], "rescans do not reprocess an unchanged file")234 w.drop("a.txt", "version 2 is longer\n")235 #expect(await eventually { w.bed.temp.exists("outbox/a 2.md") })236 #expect(w.bed.temp.read("outbox/a 2.md") == "VERSION 2 IS LONGER\n")237 #expect(await w.activity(.duplicate).isEmpty)238 await w.watcher.stop()239 }240241 @Test func identicalContentDroppedAgainIsReportedAsDuplicate() async throws {242 let w = WatchBed()243 try await w.watcher.addRule(w.rule())244 await w.watcher.start()245 w.drop("first.txt", "same bytes\n")246 #expect(await eventually { w.bed.temp.exists("outbox/first.md") })247 w.drop("copy of first.txt", "same bytes\n")248 #expect(await eventually { await w.activity(.duplicate).count == 1 })249 #expect(await w.activity(.duplicate).first?.path.hasSuffix("copy of first.txt") == true)250 // Same path removed and dropped again with the same content.251 try FileManager.default.removeItem(at: w.inbox.appendingPathComponent("first.txt"))252 try await Task.sleep(nanoseconds: 1_100_000_000)253 w.drop("first.txt", "same bytes\n")254 #expect(await eventually { await w.activity(.duplicate).count == 2 })255 #expect(w.bed.temp.list("outbox") == ["first.md"])256 await w.watcher.stop()257 }258259 @Test func duplicateContentCheckCanBeTurnedOff() async throws {260 let bed = TestBed()261 let watcher = FolderWatcher(runner: WorkflowRunner(queue: bed.queue, workDirectories: bed.workDirectories), presets: PresetStore(fileURL: bed.temp.file("p.json")),262 workflows: WorkflowLibrary(fileURL: bed.temp.file("w.json")), storeURL: bed.temp.file("watch.json"),263 configuration: WatcherConfiguration(settleInterval: 0.2, tickInterval: 0.05, rescanInterval: 0.2, skipDuplicateContent: false))264 let inbox = bed.temp.subdirectory("inbox"), outbox = bed.temp.subdirectory("outbox")265 try await watcher.addRule(WatchRule(name: "r", folder: inbox, match: .formats([.txt]), action: .convert(destination: .markdown, operation: nil, options: .init()), destinationFolder: outbox))266 await watcher.start()267 bed.temp.write("inbox/a.txt", "same\n")268 bed.temp.write("inbox/b.txt", "same\n")269 #expect(await eventually { bed.temp.list("outbox") == ["a.md", "b.md"] })270 await watcher.stop()271 }272273 @Test func pausesAndResumesARule() async throws {274 let w = WatchBed()275 let rule = try await w.watcher.addRule(w.rule())276 await w.watcher.start()277 try await w.watcher.setRulePaused(rule.id, true)278 #expect(await w.watcher.rule(id: rule.id)?.isPaused == true)279 w.drop("while-paused.txt")280 await w.settle()281 #expect(w.bed.temp.list("outbox").isEmpty)282 try await w.watcher.setRulePaused(rule.id, false)283 w.drop("after-resume.txt", "different\n")284 #expect(await eventually { w.bed.temp.exists("outbox/after-resume.md") })285 await w.settle()286 #expect(w.bed.temp.list("outbox") == ["after-resume.md"], "what arrived during the pause is not processed afterwards")287 await w.watcher.stop()288 }289290 @Test func pausesAndResumesEverything() async throws {291 let w = WatchBed()292 try await w.watcher.addRule(w.rule())293 await w.watcher.start()294 await w.watcher.pauseAll()295 #expect(await w.watcher.isPaused)296 w.drop("held.txt")297 await w.settle()298 #expect(w.bed.temp.list("outbox").isEmpty)299 await w.watcher.resumeAll()300 w.drop("released.txt", "other\n")301 #expect(await eventually { w.bed.temp.exists("outbox/released.md") })302 #expect(await w.watcher.rules.first?.isPaused == false, "the rule's own flag is untouched")303 await w.watcher.stop()304 }305306 @Test func nothingHappensBeforeStartOrAfterStop() async throws {307 let w = WatchBed()308 try await w.watcher.addRule(w.rule())309 w.drop("before.txt")310 await w.settle()311 #expect(w.bed.temp.list("outbox").isEmpty)312 await w.watcher.start()313 await w.watcher.stop()314 w.drop("after.txt", "z\n")315 await w.settle()316 #expect(w.bed.temp.list("outbox").isEmpty)317 }318319 @Test func rulesActivityAndFingerprintsPersist() async throws {320 let w = WatchBed()321 let rule = try await w.watcher.addRule(w.rule())322 await w.watcher.start()323 w.drop("a.txt", "persist me\n")324 #expect(await eventually { await w.activity(.processed).count == 1 })325 await w.watcher.stop()326327 let again = w.relaunched()328 let restored = try #require(await again.rules.first)329 #expect(restored.id == rule.id && restored.name == rule.name && restored.folderBookmark != nil && restored.destinationBookmark != nil)330 #expect(await again.activity.count == 1)331 var existing = restored332 existing.processExistingFiles = true333 try await again.updateRule(existing)334 await again.start()335 w.drop("b.txt", "new after relaunch\n")336 #expect(await eventually { w.bed.temp.exists("outbox/b.md") })337 await w.settle()338 #expect(w.bed.temp.list("outbox") == ["a.md", "b.md"], "a.txt was remembered and not converted again")339 try await again.clearActivity()340 #expect(await again.activity.isEmpty)341 await again.stop()342 }343344 @Test func rulesFollowARenamedFolderThroughTheirBookmark() async throws {345 let w = WatchBed()346 try await w.watcher.addRule(w.rule())347 let moved = w.bed.temp.file("inbox-renamed")348 try FileManager.default.moveItem(at: w.bed.temp.file("inbox"), to: moved)349 let again = w.relaunched()350 await again.start()351 #expect(await again.rules.first?.folder.lastPathComponent == "inbox-renamed")352 w.drop("x.txt", "moved\n", into: "inbox-renamed")353 #expect(await eventually { w.bed.temp.exists("outbox/x.md") })354 await again.stop()355 }356357 @Test func rulesCanRunPresetsAndWorkflows() async throws {358 let w = WatchBed()359 let preset = try await w.presets.add(ConversionPreset(name: "Loud", source: .format(.txt), destinationFormat: .markdown, options: ConversionOptions(["suffix": .string("!!")])))360 let special = w.bed.temp.subdirectory("special")361 let workflow = Workflow(name: "Sorted", nodes: [362 .branch(condition: .nameMatches("special*"), then: [.convert(destination: .yaml, operation: nil, options: .init()), .export(folder: special, conflictPolicy: .rename)], otherwise: []),363 .convert(destination: .markdown, operation: nil, options: .init()),364 ])365 try await w.workflows.save(workflow)366 let presetInbox = w.bed.temp.subdirectory("preset-inbox"), flowInbox = w.bed.temp.subdirectory("flow-inbox")367 try await w.watcher.addRule(WatchRule(name: "preset", folder: presetInbox, match: .formats([.txt]), action: .preset(preset.id), destinationFolder: w.bed.temp.subdirectory("preset-out")))368 try await w.watcher.addRule(WatchRule(name: "flow", folder: flowInbox, match: .formats([.txt]), action: .workflow(workflow.id), destinationFolder: w.bed.temp.subdirectory("flow-out")))369 await w.watcher.start()370 w.drop("p.txt", "quiet\n", into: "preset-inbox")371 w.drop("special-1.txt", "s\n", into: "flow-inbox")372 w.drop("plain.txt", "p\n", into: "flow-inbox")373 #expect(await eventually { w.bed.temp.read("preset-out/p.md") == "QUIET\n!!" })374 // The workflow's own export node decides for the special file; the path without export uses the rule's folder.375 #expect(await eventually { w.bed.temp.exists("special/special-1.yaml") && w.bed.temp.exists("flow-out/plain.md") })376 await w.settle()377 #expect(w.bed.temp.list("flow-out") == ["plain.md"] && w.bed.temp.list("special") == ["special-1.yaml"])378 await w.watcher.stop()379 }380381 @Test func invalidRulesAreRefused() async throws {382 let w = WatchBed()383 await #expect(throws: WatchError.self) { try await w.watcher.addRule(w.rule(folder: w.bed.temp.file("does-not-exist"))) }384 await #expect(throws: WatchError.self) {385 try await w.watcher.addRule(WatchRule(name: "ghost preset", folder: w.inbox, match: .any, action: .preset(UUID()), destinationFolder: w.outbox))386 }387 await #expect(throws: WatchError.self) {388 try await w.watcher.addRule(WatchRule(name: "ghost flow", folder: w.inbox, match: .any, action: .workflow(UUID()), destinationFolder: w.outbox))389 }390 await #expect(throws: WatchError.self) { try await w.watcher.removeRule(id: UUID()) }391 await #expect(throws: WatchError.self) { try await w.watcher.updateRule(w.rule()) }392 await #expect(throws: WatchError.self) { try await w.watcher.setRulePaused(UUID(), true) }393 #expect(await w.watcher.rules.isEmpty)394 let created = try await w.watcher.addRule(w.rule(destination: w.bed.temp.file("made/on/demand")))395 #expect(w.bed.temp.exists("made/on/demand"), "the destination folder is created")396 try await w.watcher.removeRule(id: created.id)397 #expect(await w.watcher.rules.isEmpty)398 }399400 @Test func failuresAreRecordedAndDoNotStopTheWatcher() async throws {401 let w = WatchBed()402 try await w.watcher.addRule(w.rule("fails", to: .rtf))403 await w.watcher.start()404 w.drop("bad.txt", "one\n")405 #expect(await eventually { await w.activity(.failed).count == 1 })406 #expect(await w.activity(.failed).first?.detail.contains("mock failure") == true)407 w.drop("bad2.txt", "two\n")408 #expect(await eventually { await w.activity(.failed).count == 2 })409 await w.settle()410 #expect(await w.watcher.activity.count == 2, "a failed file is not retried in a loop")411 await w.watcher.stop()412 }413414 @Test func eventsAnnounceActivityAndState() async throws {415 let w = WatchBed()416 let stream = await w.watcher.events()417 let collector = Task { () -> (sawRules: Bool, sawRunning: Bool, activity: WatchActivity?) in418 var sawRules = false, sawRunning = false419 for await event in stream {420 switch event {421 case .rulesChanged(let rules): sawRules = rules.count == 1422 case .stateChanged(let running, _): if running { sawRunning = true }423 case .activity(let activity): return (sawRules, sawRunning, activity)424 }425 }426 return (sawRules, sawRunning, nil)427 }428 try await w.watcher.addRule(w.rule())429 await w.watcher.start()430 w.drop("e.txt")431 let result = await collector.value432 #expect(result.sawRules && result.sawRunning && result.activity?.kind == .processed)433 await w.watcher.stop()434 }435436 @Test func activityLogIsBounded() async throws {437 let bed = TestBed()438 let watcher = FolderWatcher(runner: WorkflowRunner(queue: bed.queue, workDirectories: bed.workDirectories), presets: PresetStore(fileURL: bed.temp.file("p.json")),439 workflows: WorkflowLibrary(fileURL: bed.temp.file("w.json")), storeURL: bed.temp.file("watch.json"),440 configuration: WatcherConfiguration(settleInterval: 0.1, tickInterval: 0.03, rescanInterval: 0.1, maxActivityEntries: 10))441 let inbox = bed.temp.subdirectory("inbox")442 try await watcher.addRule(WatchRule(name: "r", folder: inbox, match: .formats([.txt]), action: .convert(destination: .markdown, operation: nil, options: .init()),443 destinationFolder: bed.temp.subdirectory("outbox")))444 await watcher.start()445 for i in 0..<16 { bed.temp.write("inbox/f\(i).txt", "content \(i)\n") }446 #expect(await eventually { bed.temp.list("outbox").count == 16 })447 #expect(await eventually { await watcher.activity.count == 10 })448 await watcher.stop()449 }450}451