Repository Public License
1// ORIX Convert — Packages/ORIXConvertCore/Tests/ConvertCoreTests/QueueTests.swift2// Description: The conversion queue end to end over the mock engine: states, chained plans, validation failures,3// conflicts, cancellation, retry, priorities, pause, resource governance, events, ETA and history.4// Author: Simon-Pierre Boucher — contact@spboucher.ai5import Foundation6import Testing7@testable import ConvertCore89@Suite struct QueueConversionTests {10 @Test func convertsAFileEndToEnd() async throws {11 let bed = TestBed()12 let source = bed.source("note.txt", "hello world\n")13 let id = await bed.queue.enqueue(ConversionJobRequest(sources: [source], destinationFormat: .markdown, destination: bed.destination()))14 let job = try #require(await bed.queue.waitForJob(id))15 #expect(job.state == .finished)16 #expect(job.sourceFormat == .txt && job.detection?.format == .txt)17 #expect(job.plan?.steps.count == 1 && job.plan?.engines == ["mock-text"])18 #expect(job.outputs.map(\.url.lastPathComponent) == ["note.md"])19 #expect(bed.temp.read("out/note.md") == "HELLO WORLD\n")20 #expect(job.checks.allPassed && job.checks.contains { $0.name == "output.format" } && job.checks.contains { $0.name == "mock.reopen" })21 #expect(job.facts["characters"] == "12")22 #expect(job.progress.fraction == 1 && job.bytesBefore == 12 && job.bytesAfter == 12)23 #expect(job.startedAt != nil && job.finishedAt != nil && job.duration != nil)24 #expect(bed.temp.read("in/note.txt") == "hello world\n", "the source is untouched")25 #expect(bed.leftoverWorkDirectories.isEmpty, "the work directory is cleaned")26 }2728 @Test func writesNextToTheSourceByDefault() async throws {29 let bed = TestBed()30 let source = bed.source()31 let id = await bed.queue.enqueue(ConversionJobRequest(source: source, to: .markdown))32 let job = try #require(await bed.queue.waitForJob(id))33 #expect(job.outputs.first?.url.deletingLastPathComponent().lastPathComponent == "in")34 }3536 @Test func jobLogIsTimestampedAndOrdered() async throws {37 let bed = TestBed()38 let id = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .markdown, destination: bed.destination()))39 let job = try #require(await bed.queue.waitForJob(id))40 let messages = job.log.map(\.message)41 #expect(messages.first == "Queued." && messages.last == "Finished.")42 #expect(messages.contains { $0.hasPrefix("Detected txt") })43 #expect(messages.contains { $0.hasPrefix("Plan: txt → markdown") })44 #expect(messages.contains("mock converting note.txt"), "engine log lines are collected")45 #expect(messages.contains("Limitation: Text is upper-cased."))46 #expect(zip(job.log, job.log.dropFirst()).allSatisfy { $0.date <= $1.date })47 }4849 @Test func passesOptionsToTheEngine() async throws {50 let bed = TestBed()51 let request = ConversionJobRequest(sources: [bed.source("a.txt", "x")], destinationFormat: .markdown, options: ConversionOptions(["suffix": .string("!")]),52 destination: bed.destination())53 _ = await bed.queue.waitForJob(await bed.queue.enqueue(request))54 #expect(bed.temp.read("out/a.md") == "X!")55 }5657 @Test func executesAChainedPlanThroughAnIntermediate() async throws {58 let bed = TestBed()59 let request = ConversionJobRequest(sources: [bed.source("page.txt", "body")], destinationFormat: .html, options: ConversionOptions(["suffix": .string("?")]),60 destination: bed.destination())61 let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(request)))62 #expect(job.state == .finished)63 #expect(job.plan?.isChained == true && job.plan?.intermediates == [.markdown])64 #expect(bed.temp.list("out") == ["page.html"], "the intermediate never reaches the destination")65 // Step 1 ran with default options (no suffix); step 2 received the upper-cased text.66 #expect(bed.temp.read("out/page.html").contains("<pre>BODY</pre>"))67 #expect(job.checks.contains { $0.name.hasPrefix("step1.") } && job.checks.contains { $0.name.hasPrefix("step2.") })68 #expect(job.log.contains { $0.message.hasPrefix("Step 2 of 2") })69 #expect(bed.leftoverWorkDirectories.isEmpty)70 }7172 @Test func chainingCanBeRefusedPerJob() async throws {73 let bed = TestBed()74 var request = ConversionJobRequest(sources: [bed.source()], destinationFormat: .html, destination: bed.destination())75 request.allowChaining = false76 let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(request)))77 #expect(job.state == .failed && job.failure == .unsupportedPath(source: .txt, destination: .html))78 }7980 @Test func detectsTheFormatFromContentNotFromTheName() async throws {81 let bed = TestBed()82 let source = bed.source("misnamed.png", "really just text\n")83 let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(sources: [source], destinationFormat: .markdown, destination: bed.destination()))))84 #expect(job.state == .finished && job.sourceFormat == .txt)85 #expect(job.warnings.contains { $0.code == "extension-mismatch" })86 }8788 @Test func unknownFormatAndUnsupportedPathFailCleanly() async throws {89 let bed = TestBed()90 let blob = bed.temp.write("in/blob.xyz", bytes: [0xDE, 0xAD, 0xBE, 0xEF, 0, 1, 2, 3])91 let unknown = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(source: blob, to: .markdown))))92 #expect(unknown.state == .failed)93 if case .malformedInput = unknown.failure {} else { Issue.record("expected malformedInput, got \(String(describing: unknown.failure))") }94 let unsupported = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(source: bed.source(), to: .mp3))))95 #expect(unsupported.failure == .unsupportedPath(source: .txt, destination: .mp3))96 }9798 @Test func missingSourceFails() async throws {99 let bed = TestBed()100 let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(source: bed.temp.file("in/ghost.txt"), to: .markdown))))101 #expect(job.state == .failed)102 if case .unreadableInput = job.failure {} else { Issue.record("expected unreadableInput") }103 }104105 @Test func engineErrorsFailOnlyThatJob() async throws {106 let bed = TestBed()107 let ids = await bed.queue.enqueue(batch: [108 ConversionJobRequest(sources: [bed.source("a.txt")], destinationFormat: .markdown, destination: bed.destination()),109 ConversionJobRequest(sources: [bed.source("b.txt")], destinationFormat: .rtf, destination: bed.destination()),110 ConversionJobRequest(sources: [bed.source("c.txt")], destinationFormat: .markdown, destination: bed.destination()),111 ])112 await bed.queue.waitUntilIdle()113 let jobs = await bed.queue.snapshot().jobs114 #expect(jobs.map(\.state) == [.finished, .failed, .finished])115 #expect(jobs[1].failure == .malformedInput("mock failure") && jobs[1].error?.contains("mock failure") == true)116 #expect(Set(jobs.compactMap(\.request.batchID)).count == 1 && jobs.map(\.request.counter) == [1, 2, 3])117 #expect(bed.temp.list("out") == ["a.md", "c.md"])118 #expect(ids.count == 3 && bed.leftoverWorkDirectories.isEmpty)119 }120121 @Test func outputOfTheWrongFormatIsRejectedAndNeverDelivered() async throws {122 let bed = TestBed()123 let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .png, destination: bed.destination()))))124 #expect(job.state == .failed)125 if case .outputInvalid(let message) = job.failure { #expect(message.contains("png")) } else { Issue.record("expected outputInvalid") }126 #expect(job.checks.contains { $0.name == "output.format" && !$0.passed })127 #expect(bed.temp.list("out").isEmpty && bed.leftoverWorkDirectories.isEmpty)128 }129130 @Test func emptyOutputIsRejected() async throws {131 let bed = TestBed()132 let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .csv, destination: bed.destination()))))133 #expect(job.state == .failed && job.checks.contains { $0.name == "output.not-empty" && !$0.passed })134 #expect(bed.temp.list("out").isEmpty)135 }136137 @Test func engineValidationFailureBlocksDelivery() async throws {138 let bed = TestBed()139 let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .toml, destination: bed.destination()))))140 #expect(job.state == .failed && job.failure == .outputInvalid("The mock refuses its TOML output."))141 #expect(bed.temp.list("out").isEmpty)142 }143144 @Test func directoryOutputWithEscapingSymlinkIsRejected() async throws {145 let bed = TestBed()146 let request = ConversionJobRequest(sources: [bed.source()], destinationFormat: .folder, operation: "text.escape", destination: bed.destination())147 let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(request)))148 #expect(job.state == .failed && job.checks.contains { $0.name == "output.no-symlink-escape" && !$0.passed })149 #expect(bed.temp.list("out").isEmpty)150 }151152 @Test func directoryOutputIsDelivered() async throws {153 let bed = TestBed()154 let request = ConversionJobRequest(sources: [bed.source("list.txt", "a\nb\nc\n")], destinationFormat: .folder, operation: "text.explode", destination: bed.destination())155 let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(request)))156 #expect(job.state == .finished)157 #expect(bed.temp.list("out/list") == ["line-1.txt", "line-2.txt", "line-3.txt"])158 #expect(job.bytesAfter == 3)159 }160161 @Test func multipleOutputsAreAllDelivered() async throws {162 let bed = TestBed()163 let request = ConversionJobRequest(sources: [bed.source("rows.txt", "a\nb\n")], destinationFormat: .tsv, destination: bed.destination(template: "{name}_out"))164 let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(request)))165 // Several files for one job: ONE folder named like the output, holding them (never loose files).166 #expect(job.state == .finished && bed.temp.list("out") == ["rows_out"] && bed.temp.list("out/rows_out") == ["rows_out-1.tsv", "rows_out-2.tsv"])167 #expect(job.outputs.map(\.url.lastPathComponent) == ["rows_out"] && job.facts["outputFiles"] == "2")168 // Asked not to group (what workflows do): the loose files.169 var loose = ConversionJobRequest(sources: [bed.source("rows.txt", "a\nb\n")], destinationFormat: .tsv, destination: bed.destination(template: "{name}_loose"))170 loose.groupMultipleOutputs = false171 let second = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(loose)))172 #expect(second.outputs.map(\.url.lastPathComponent) == ["rows_loose-1.tsv", "rows_loose-2.tsv"])173 }174175 @Test func multiSourceOperation() async throws {176 let bed = TestBed()177 let sources = [bed.source("1.txt", "one"), bed.source("2.txt", "two"), bed.source("3.txt", "three")]178 let merge = ConversionJobRequest(sources: sources, destinationFormat: .markdown, operation: "text.merge", destination: bed.destination())179 let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(merge)))180 #expect(job.state == .finished && bed.temp.read("out/1.md") == "one\ntwo\nthree" && job.bytesBefore == 11)181 // A single-source path refuses several inputs instead of ignoring the extra ones.182 let wrong = ConversionJobRequest(sources: sources, destinationFormat: .markdown, destination: bed.destination())183 let failed = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(wrong)))184 if case .invalidOption = failed.failure {} else { Issue.record("expected invalidOption") }185 }186187 @Test func warningsFromEngineAndPreflightAreCollected() async throws {188 let bed = TestBed()189 let request = ConversionJobRequest(sources: [bed.source()], destinationFormat: .latex, options: ConversionOptions(["warn": .bool(true)]), destination: bed.destination())190 let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(request)))191 #expect(job.state == .finished)192 #expect(job.warnings.map(\.code).sorted() == ["preflight-test", "styles-lost"])193 #expect(job.warnings.contains { $0.severity == .dataLoss })194 #expect(job.log.contains { $0.level == .warning && $0.message == "Styles were dropped." })195 }196197 @Test func nameTemplateUsesMeasuredOutputDimensions() async throws {198 let bed = TestBed()199 let request = ConversionJobRequest(sources: [bed.source("img.txt", "abc")], destinationFormat: .latex, destination: bed.destination(template: "{name}_{width}x{height}"), counter: 4)200 let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(request)))201 // The source inspection says 3x1; the engine measured 640x480 on its output.202 #expect(job.outputs.map(\.url.lastPathComponent) == ["img_640x480.tex"])203 let counter = ConversionJobRequest(sources: [bed.source("img.txt", "abc")], destinationFormat: .markdown, destination: bed.destination(template: "{counter:2}-{name}-{ext}"), counter: 4)204 let second = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(counter)))205 #expect(second.outputs.map(\.url.lastPathComponent) == ["04-img-txt.md"])206 }207208 @Test func forcedEngineIsRespected() async throws {209 var other = TextEngine()210 other.id = "mock-b"211 let bed = TestBed(extraEngines: [other])212 var request = ConversionJobRequest(sources: [bed.source()], destinationFormat: .markdown, destination: bed.destination())213 request.engineID = "mock-b"214 let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(request)))215 #expect(job.state == .finished && job.plan?.engines == ["mock-b"])216 }217}218219@Suite struct QueueConflictTests {220 @Test func renamePolicyKeepsBoth() async throws {221 let bed = TestBed()222 bed.temp.write("out/note.md", "existing")223 let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .markdown, destination: bed.destination(.rename)))))224 #expect(job.outputs.map(\.outcome) == [.renamed] && bed.temp.list("out") == ["note 2.md", "note.md"])225 #expect(bed.temp.read("out/note.md") == "existing")226 }227228 @Test func skipPolicySkipsBeforeConverting() async throws {229 let bed = TestBed()230 bed.temp.write("out/note.md", "existing")231 let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .markdown, destination: bed.destination(.skip)))))232 #expect(job.state == .skipped && job.outputs.isEmpty && bed.temp.read("out/note.md") == "existing")233 #expect(bed.probe.order.isEmpty, "nothing was converted")234 }235236 @Test func replacePolicyReplaces() async throws {237 let bed = TestBed()238 bed.temp.write("out/note.md", "existing")239 let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("note.txt", "new")], destinationFormat: .markdown, destination: bed.destination(.replace)))))240 #expect(job.outputs.map(\.outcome) == [.replaced] && bed.temp.read("out/note.md") == "NEW")241 }242243 @Test func askUserSurfacesADecisionBeforeConverting() async throws {244 let bed = TestBed()245 bed.temp.write("out/note.md", "existing")246 let id = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("note.txt", "new")], destinationFormat: .markdown, destination: bed.destination(.askUser)))247 let waiting = try #require(await bed.queue.waitForJob(id))248 #expect(waiting.state == .awaitingDecision)249 #expect(waiting.conflict?.existing.lastPathComponent == "note.md" && waiting.conflict?.conversionDone == false && waiting.conflict?.isSource == false)250 #expect(bed.probe.order.isEmpty && bed.temp.read("out/note.md") == "existing")251 #expect(await bed.queue.resolveConflict(id, policy: .askUser) == false, "asking again is not a decision")252 #expect(await bed.queue.resolveConflict(id, policy: .rename))253 let done = try #require(await bed.queue.waitForJob(id))254 #expect(done.state == .finished && done.conflict == nil && bed.temp.read("out/note 2.md") == "NEW")255 }256257 @Test func askUserDecisionCanBeSkipOrReplace() async throws {258 let bed = TestBed()259 bed.temp.write("out/a.md", "existing")260 bed.temp.write("out/b.md", "existing")261 let a = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("a.txt", "new")], destinationFormat: .markdown, destination: bed.destination(.askUser)))262 let b = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("b.txt", "new")], destinationFormat: .markdown, destination: bed.destination(.askUser)))263 _ = await bed.queue.waitForJob(a)264 _ = await bed.queue.waitForJob(b)265 #expect(await bed.queue.resolveConflict(a, policy: .skip))266 #expect(await bed.queue.resolveConflict(b, policy: .replace))267 #expect(await bed.queue.waitForJob(a)?.state == .skipped)268 #expect(await bed.queue.waitForJob(b)?.state == .finished)269 #expect(bed.temp.read("out/a.md") == "existing" && bed.temp.read("out/b.md") == "NEW")270 }271272 @Test func conflictAppearingDuringConversionHoldsTheOutput() async throws {273 let bed = TestBed(steps: 20, stepDelay: 0.02)274 let id = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("slow.txt")], destinationFormat: .yaml, destination: bed.destination(.askUser)))275 #expect(await eventually { await bed.queue.job(id)?.state == .running })276 bed.temp.write("out/slow.yaml", "appeared: meanwhile\n")277 let waiting = try #require(await bed.queue.waitForJob(id))278 #expect(waiting.state == .awaitingDecision && waiting.conflict?.conversionDone == true)279 #expect(bed.leftoverWorkDirectories.count == 1, "the converted file is held in its work directory")280 #expect(await bed.queue.snapshot().status.active == 0, "a waiting job holds no slot")281 #expect(await bed.queue.resolveConflict(id, policy: .rename))282 let done = try #require(await bed.queue.waitForJob(id))283 #expect(done.state == .finished && bed.temp.exists("out/slow 2.yaml") && bed.temp.read("out/slow.yaml") == "appeared: meanwhile\n")284 #expect(bed.leftoverWorkDirectories.isEmpty && bed.probe.order.count == 1, "resolving does not convert again")285 }286287 @Test func cancellingAHeldJobCleansItsWorkDirectory() async throws {288 let bed = TestBed(steps: 10, stepDelay: 0.02)289 let id = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("slow.txt")], destinationFormat: .yaml, destination: bed.destination(.askUser)))290 #expect(await eventually { await bed.queue.job(id)?.state == .running })291 bed.temp.write("out/slow.yaml", "x: 1\n")292 #expect(await bed.queue.waitForJob(id)?.state == .awaitingDecision)293 #expect(await bed.queue.cancel(id))294 #expect(await bed.queue.waitForJob(id)?.state == .cancelled)295 #expect(bed.leftoverWorkDirectories.isEmpty)296 }297298 @Test func neverOverwritesTheSourceInPlace() async throws {299 let bed = TestBed()300 // txt → markdown named ".txt"? Use a markdown-named text source so the output name collides with the source.301 let source = bed.temp.write("in/doc.md", "plain words only\n")302 var request = ConversionJobRequest(sources: [source], sourceFormat: .txt, destinationFormat: .markdown, destination: OutputDestination(conflictPolicy: .replace))303 let refused = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(request)))304 #expect(refused.state == .failed)305 if case .destinationExists = refused.failure {} else { Issue.record("expected destinationExists") }306 #expect(bed.temp.read("in/doc.md") == "plain words only\n" && bed.probe.order.isEmpty)307308 request.destination = OutputDestination(conflictPolicy: .rename)309 let renamed = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(request)))310 #expect(renamed.outputs.map(\.url.lastPathComponent) == ["doc 2.md"] && bed.temp.read("in/doc.md") == "plain words only\n")311312 request.destination = OutputDestination(conflictPolicy: .askUser)313 let asked = await bed.queue.enqueue(request)314 #expect(await bed.queue.waitForJob(asked)?.conflict?.isSource == true)315 #expect(await bed.queue.resolveConflict(asked, policy: .replace), "accepted, the job is converted…")316 let stillRefused = try #require(await bed.queue.waitForJob(asked))317 #expect(stillRefused.state == .failed, "…but replacing the source needs the explicit flag")318 #expect(bed.temp.read("in/doc.md") == "plain words only\n")319320 request.destination = OutputDestination(conflictPolicy: .replace, allowReplacingSource: true)321 let replaced = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(request)))322 #expect(replaced.state == .finished && bed.temp.read("in/doc.md") == "PLAIN WORDS ONLY\n")323 }324}325326@Suite struct QueueControlTests {327 @Test func cancelsARunningJob() async throws {328 let bed = TestBed(steps: 500, stepDelay: 0.02)329 let id = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .yaml, destination: bed.destination()))330 #expect(await eventually { await bed.queue.job(id)?.state == .running })331 let started = Date()332 #expect(await bed.queue.cancel(id))333 #expect(await bed.queue.job(id)?.cancelRequested == true)334 let job = try #require(await bed.queue.waitForJob(id))335 #expect(job.state == .cancelled && Date().timeIntervalSince(started) < 3, "the engine stopped promptly, long before its 10 s run")336 #expect(bed.temp.list("out").isEmpty && bed.leftoverWorkDirectories.isEmpty)337 #expect(await bed.queue.cancel(id) == false, "already settled")338 }339340 @Test func cancelsAHelperProcessThroughProcessRunner() async throws {341 struct SleepEngine: ConversionEngine {342 let id = "sleep"343 let displayName = "Sleep"344 var paths: [ConversionPath] { [ConversionPath(.txt, .yaml, engine: id, fidelity: .lossless)] }345 func inspect(_ url: URL, format: FormatID) async throws -> FileInspection { FileInspection(format: format) }346 func convert(_ request: ConversionRequest, context: ConversionContext) async throws -> ConversionOutput {347 try await ProcessRunner.runChecked(URL(fileURLWithPath: "/bin/sleep"), arguments: ["30"])348 return ConversionOutput(files: [])349 }350 }351 let temp = TempDir("sleep")352 let queue = ConversionQueue(registry: FormatRegistry(engines: [SleepEngine()]), workDirectories: WorkDirectoryManager(root: temp.file("work")))353 let id = await queue.enqueue(ConversionJobRequest(source: temp.write("a.txt", "x"), to: .yaml))354 #expect(await eventually { await queue.job(id)?.state == .running })355 try await Task.sleep(nanoseconds: 200_000_000)356 let started = Date()357 await queue.cancel(id)358 #expect(await queue.waitForJob(id)?.state == .cancelled)359 #expect(Date().timeIntervalSince(started) < 5)360 }361362 @Test func cancelsQueuedJobsImmediatelyAndCancelAll() async throws {363 let bed = TestBed(maxConcurrent: 1, steps: 500)364 let ids = await bed.queue.enqueue(batch: (0..<4).map { ConversionJobRequest(sources: [bed.source("f\($0).txt")], destinationFormat: .yaml, destination: bed.destination()) })365 #expect(await eventually { await bed.queue.job(ids[0])?.state == .running })366 #expect(await bed.queue.cancel(ids[3]))367 #expect(await bed.queue.job(ids[3])?.state == .cancelled)368 await bed.queue.cancelAll()369 await bed.queue.waitUntilIdle()370 #expect(await bed.queue.snapshot().jobs.map(\.state) == [.cancelled, .cancelled, .cancelled, .cancelled])371 #expect(bed.probe.order == ["f0.txt"], "the waiting jobs never started")372 }373374 @Test func retryRunsAFailedOrCancelledJobAgain() async throws {375 let bed = TestBed(steps: 500)376 let id = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .yaml, destination: bed.destination()))377 #expect(await eventually { await bed.queue.job(id)?.state == .running })378 #expect(await bed.queue.retry(id) == false, "a running job cannot be retried")379 await bed.queue.cancel(id)380 #expect(await bed.queue.waitForJob(id)?.state == .cancelled)381382 let failing = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .rtf, destination: bed.destination()))383 #expect(await bed.queue.waitForJob(failing)?.state == .failed)384 #expect(await bed.queue.retry(failing))385 let again = try #require(await bed.queue.waitForJob(failing))386 #expect(again.state == .failed && again.attempt == 2 && again.log.contains { $0.message == "Retry (attempt 2)." })387 #expect(try await bed.history.count() == 3, "each ended attempt is recorded")388 }389390 @Test func retryAfterFixingTheCauseSucceeds() async throws {391 let bed = TestBed()392 let source = bed.temp.file("in/late.txt")393 let id = await bed.queue.enqueue(ConversionJobRequest(sources: [source], sourceFormat: .txt, destinationFormat: .markdown, destination: bed.destination()))394 #expect(await bed.queue.waitForJob(id)?.state == .failed)395 bed.temp.write("in/late.txt", "now here")396 #expect(await bed.queue.retry(id))397 let job = try #require(await bed.queue.waitForJob(id))398 #expect(job.state == .finished && job.failure == nil && job.attempt == 2)399 }400401 @Test func queuePauseHoldsNewStartsAndLetsRunningJobsFinish() async throws {402 let bed = TestBed(maxConcurrent: 1, steps: 10)403 let first = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("a.txt")], destinationFormat: .yaml, destination: bed.destination()))404 let second = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("b.txt")], destinationFormat: .markdown, destination: bed.destination()))405 #expect(await eventually { await bed.queue.job(first)?.state == .running })406 await bed.queue.pause()407 #expect(await bed.queue.isPaused)408 #expect(await bed.queue.waitForJob(first)?.state == .finished, "a running job is not interrupted by pausing the queue")409 await bed.queue.waitUntilIdle()410 #expect(await bed.queue.job(second)?.state == .queued)411 await bed.queue.resume()412 #expect(await bed.queue.waitForJob(second)?.state == .finished)413 }414415 @Test func jobPauseAppliesBeforeStartOnly() async throws {416 let bed = TestBed(maxConcurrent: 1, steps: 15)417 let running = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("a.txt")], destinationFormat: .yaml, destination: bed.destination()))418 let waiting = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("b.txt")], destinationFormat: .markdown, destination: bed.destination()))419 #expect(await eventually { await bed.queue.job(running)?.state == .running })420 #expect(await bed.queue.pauseJob(running) == false, "no cooperative suspension: a running job is not paused")421 #expect(await bed.queue.pauseJob(waiting))422 #expect(await bed.queue.waitForJob(running)?.state == .finished)423 await bed.queue.waitUntilIdle()424 #expect(await bed.queue.job(waiting)?.state == .paused && bed.probe.order == ["a.txt"])425 #expect(await bed.queue.resumeJob(waiting))426 #expect(await bed.queue.waitForJob(waiting)?.state == .finished)427 #expect(await bed.queue.resumeJob(waiting) == false)428 }429430 @Test func higherPriorityStartsFirst() async throws {431 let bed = TestBed(maxConcurrent: 1)432 await bed.queue.pause()433 func request(_ name: String, _ priority: JobPriority) -> ConversionJobRequest {434 ConversionJobRequest(sources: [bed.source(name)], destinationFormat: .markdown, destination: bed.destination(), priority: priority)435 }436 await bed.queue.enqueue(request("low.txt", .low))437 await bed.queue.enqueue(request("normal1.txt", .normal))438 await bed.queue.enqueue(request("high.txt", .high))439 await bed.queue.enqueue(request("normal2.txt", .normal))440 await bed.queue.resume()441 await bed.queue.waitUntilIdle()442 #expect(bed.probe.order == ["high.txt", "normal1.txt", "normal2.txt", "low.txt"])443 }444445 @Test func prioritiesAndOrderCanBeChangedWhileWaiting() async throws {446 let bed = TestBed(maxConcurrent: 1)447 await bed.queue.pause()448 let ids = await bed.queue.enqueue(batch: ["a", "b", "c", "d"].map { ConversionJobRequest(sources: [bed.source($0 + ".txt")], destinationFormat: .markdown, destination: bed.destination()) })449 #expect(await bed.queue.setPriority(.high, for: ids[2]))450 #expect(await bed.queue.move(ids[3], to: 0))451 #expect(await bed.queue.snapshot().jobs.map(\.id) == [ids[3], ids[0], ids[1], ids[2]])452 await bed.queue.resume()453 await bed.queue.waitUntilIdle()454 #expect(bed.probe.order == ["c.txt", "d.txt", "a.txt", "b.txt"])455 #expect(await bed.queue.setPriority(.low, for: ids[0]) == false, "a finished job has no priority to change")456 }457458 @Test func removeAndClearSettled() async throws {459 let bed = TestBed(maxConcurrent: 1, steps: 500)460 let running = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("a.txt")], destinationFormat: .yaml, destination: bed.destination()))461 let waiting = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("b.txt")], destinationFormat: .markdown, destination: bed.destination()))462 #expect(await eventually { await bed.queue.job(running)?.state == .running })463 #expect(await bed.queue.remove(running) == false, "running jobs must be cancelled first")464 #expect(await bed.queue.remove(waiting))465 #expect(await bed.queue.job(waiting) == nil)466 await bed.queue.cancel(running)467 _ = await bed.queue.waitForJob(running)468 #expect(await bed.queue.clearSettled() == 1)469 #expect(await bed.queue.snapshot().jobs.isEmpty)470 #expect(await bed.queue.waitForJob(UUID()) == nil)471 }472}473474@Suite struct QueueGovernanceTests {475 private func runBatch(_ bed: TestBed, count: Int, to format: FormatID = .yaml) async {476 await bed.queue.enqueue(batch: (0..<count).map { ConversionJobRequest(sources: [bed.source("f\($0).txt")], destinationFormat: format, destination: bed.destination()) })477 await bed.queue.waitUntilIdle()478 }479480 @Test func respectsTheConcurrencyLimit() async throws {481 let bed = TestBed(maxConcurrent: 3, steps: 5)482 await runBatch(bed, count: 10)483 #expect(bed.probe.maximum == 3)484 #expect(await bed.queue.snapshot().jobs.allSatisfy { $0.state == .finished })485 }486487 @Test func heavyJobsHaveFewerSlotsAndDoNotBlockLightOnes() async throws {488 let bed = TestBed(maxConcurrent: 4, maxHeavy: 1, steps: 8, heavyEngines: ["mock-text"])489 var heavy = (0..<3).map { ConversionJobRequest(sources: [bed.source("h\($0).txt")], destinationFormat: .yaml, destination: bed.destination()) }490 for index in heavy.indices { heavy[index].sourceFormat = .txt }491 let ids = await bed.queue.enqueue(batch: heavy)492 #expect(await bed.queue.job(ids[0])?.isHeavy == true)493 await bed.queue.waitUntilIdle()494 #expect(bed.probe.maximum == 1, "one heavy slot")495 }496497 @Test func heavyClassification() async throws {498 let bed = TestBed(heavyEngines: ["mock-text"])499 let light = FormatRegistry(engines: [TextEngine(), HTMLEngine()])500 let lightQueue = ConversionQueue(registry: light, configuration: QueueConfiguration(maxConcurrentJobs: 2, heavyFileSize: 10),501 workDirectories: bed.workDirectories, monitor: bed.monitor)502 await lightQueue.pause()503 let small = await lightQueue.enqueue(ConversionJobRequest(source: bed.source("s.txt", "tiny"), to: .markdown))504 let large = await lightQueue.enqueue(ConversionJobRequest(source: bed.source("l.txt", String(repeating: "x", count: 64)), to: .markdown))505 let video = await lightQueue.enqueue(ConversionJobRequest(source: bed.source("v.txt"), to: .mp4))506 #expect(await lightQueue.job(small)?.isHeavy == false)507 #expect(await lightQueue.job(large)?.isHeavy == true, "above the size threshold")508 #expect(await lightQueue.job(video)?.isHeavy == true, "video destination")509 await bed.queue.pause()510 let forced = await bed.queue.enqueue(ConversionJobRequest(source: bed.source("e.txt"), to: .markdown))511 #expect(await bed.queue.job(forced)?.isHeavy == true, "planned on a heavy engine")512 }513514 @Test func limitsFollowThermalAndPowerState() {515 let config = QueueConfiguration(maxConcurrentJobs: 8, maxHeavyJobs: 2, reduceInLowPowerMode: true, reduceOnBattery: true)516 #expect(config.limits(for: ResourceState()) == ConcurrencyLimits(total: 8, heavy: 2, reason: nil))517 #expect(config.limits(for: ResourceState(thermal: .fair)).total == 8)518 let serious = config.limits(for: ResourceState(thermal: .serious))519 #expect(serious.total == 4 && serious.heavy == 1 && serious.reason != nil)520 let critical = config.limits(for: ResourceState(thermal: .critical))521 #expect(critical.total == 0 && critical.heavy == 0)522 let lowPower = config.limits(for: ResourceState(lowPowerMode: true))523 #expect(lowPower.total == 4 && lowPower.heavy == 1 && lowPower.reason?.contains("Low Power Mode") == true)524 #expect(config.limits(for: ResourceState(onBattery: true)).total == 4)525 let optedOut = QueueConfiguration(maxConcurrentJobs: 8, reduceInLowPowerMode: false, reduceOnBattery: false)526 #expect(optedOut.limits(for: ResourceState(lowPowerMode: true, onBattery: true)).total == 8)527 #expect(QueueConfiguration(maxConcurrentJobs: 1).limits(for: ResourceState(thermal: .serious, lowPowerMode: true)).total == 1)528 #expect(QueueConfiguration(maxConcurrentJobs: 2, maxHeavyJobs: 9).limits(for: ResourceState()).heavy == 2, "heavy never exceeds total")529 }530531 @Test func defaultConcurrencyComesFromPerformanceCores() {532 let cores = QueueConfiguration.defaultConcurrency()533 #expect((1...8).contains(cores))534 #expect(QueueConfiguration().maxConcurrentJobs == cores && QueueConfiguration().maxHeavyJobs >= 1)535 #expect(QueueConfiguration(maxConcurrentJobs: 0).maxConcurrentJobs == 1)536 }537538 @Test func seriousThermalStateHalvesConcurrency() async throws {539 let bed = TestBed(maxConcurrent: 4, state: ResourceState(thermal: .serious), steps: 5)540 await runBatch(bed, count: 8)541 #expect(bed.probe.maximum == 2)542 #expect(await bed.queue.snapshot().status.limits.reason?.contains("thermal") == true)543 }544545 @Test func criticalThermalStatePausesNewStartsUntilItCools() async throws {546 let bed = TestBed(maxConcurrent: 4, state: ResourceState(thermal: .critical))547 let id = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .markdown, destination: bed.destination()))548 try await Task.sleep(nanoseconds: 300_000_000)549 #expect(await bed.queue.job(id)?.state == .queued && bed.probe.order.isEmpty)550 #expect(await bed.queue.snapshot().status.limits.total == 0)551 bed.monitor.box.mutate { $0.thermal = .nominal }552 await bed.queue.resourceStateChanged()553 #expect(await bed.queue.waitForJob(id)?.state == .finished)554 }555556 @Test func lowPowerModeReducesConcurrencyWhenEnabled() async throws {557 let bed = TestBed(maxConcurrent: 4, state: ResourceState(lowPowerMode: true), steps: 5)558 await runBatch(bed, count: 8)559 #expect(bed.probe.maximum == 2)560 }561562 @Test func configurationCanBeChangedLive() async throws {563 let bed = TestBed(maxConcurrent: 1, steps: 5)564 await bed.queue.pause()565 await bed.queue.enqueue(batch: (0..<6).map { ConversionJobRequest(sources: [bed.source("f\($0).txt")], destinationFormat: .yaml, destination: bed.destination()) })566 await bed.queue.updateConfiguration(QueueConfiguration(maxConcurrentJobs: 3))567 #expect(await bed.queue.currentConfiguration.maxConcurrentJobs == 3)568 await bed.queue.resume()569 await bed.queue.waitUntilIdle()570 #expect(bed.probe.maximum == 3)571 }572573 @Test func systemMonitorReadsRealState() {574 let state = SystemResourceMonitor().currentState()575 #expect(state.thermal >= .nominal && state.thermal <= .critical)576 _ = SystemResourceMonitor.isOnBattery()577 #expect(ThermalLevel(.serious) == .serious && ThermalLevel(.nominal) < ThermalLevel(.critical))578 }579}580581@Suite struct QueueObservationTests {582 @Test func eventsDescribeTheLifeOfAJob() async throws {583 let bed = TestBed(steps: 5)584 let stream = await bed.queue.events()585 let collector = Task { () -> [QueueEvent] in586 var events: [QueueEvent] = []587 for await event in stream {588 events.append(event)589 if case .jobChanged(let job) = event, job.state.isTerminal { break }590 }591 return events592 }593 let id = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .yaml, destination: bed.destination()))594 let events = await collector.value595 var states: [JobState] = []596 var fractions: [Double] = []597 var sawAdded = false, sawLog = false598 for event in events {599 switch event {600 case .jobAdded(let job): sawAdded = job.id == id601 case .jobChanged(let job): if states.last != job.state { states.append(job.state) }602 case .jobProgress(_, let progress, _): if let f = progress.fraction { fractions.append(f) }603 case .jobLog: sawLog = true604 default: break605 }606 }607 #expect(sawAdded && sawLog)608 #expect(states == [.preparing, .running, .validating, .finished])609 #expect(fractions == fractions.sorted() && fractions.last == 1 && fractions.count == 5, "measured progress, in order")610 }611612 @Test func severalSubscribersAndUnsubscription() async throws {613 let bed = TestBed()614 let first = await bed.queue.events()615 let second = await bed.queue.events()616 let counter = Task { () -> Int in617 var n = 0618 for await event in second { if case .jobAdded = event { n += 1; break } }619 return n620 }621 let dropped = Task { for await _ in first { break } }622 await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .markdown, destination: bed.destination()))623 #expect(await counter.value == 1)624 await dropped.value625 await bed.queue.waitUntilIdle()626 }627628 @Test func idleEventAndSnapshotStatus() async throws {629 let bed = TestBed(maxConcurrent: 2, steps: 5)630 let stream = await bed.queue.events()631 let idle = Task { () -> Bool in632 for await event in stream { if case .idle = event { return true } }633 return false634 }635 await bed.queue.enqueue(batch: (0..<3).map { ConversionJobRequest(sources: [bed.source("f\($0).txt")], destinationFormat: .yaml, destination: bed.destination()) })636 #expect(await idle.value)637 let snapshot = await bed.queue.snapshot()638 #expect(snapshot.status.active == 0 && snapshot.status.queued == 0 && snapshot.status.settled == 3 && !snapshot.status.isPaused)639 #expect(snapshot.status.limits.total == 2)640 }641642 @Test func progressOfAChainedPlanSpansAllSteps() async throws {643 let bed = TestBed()644 let stream = await bed.queue.events()645 let collector = Task { () -> [JobProgress] in646 var all: [JobProgress] = []647 for await event in stream {648 if case .jobProgress(_, let progress, _) = event { all.append(progress) }649 if case .jobChanged(let job) = event, job.state.isTerminal { break }650 }651 return all652 }653 await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .html, destination: bed.destination()))654 let progress = await collector.value655 // Step 1 (txt → markdown) reports 1.0 of step 1 of 2 → 0.5 overall; step 2 reports nothing.656 #expect(progress.map(\.fraction) == [0.5] && progress.first?.stepCount == 2 && progress.first?.stepIndex == 0)657 }658659 @Test func etaNeedsMeasuredProgress() {660 #expect(ETAEstimator.remaining(fraction: nil, elapsed: 10) == nil)661 #expect(ETAEstimator.remaining(fraction: 0, elapsed: 10) == nil)662 #expect(ETAEstimator.remaining(fraction: 0.001, elapsed: 10) == nil, "below the minimum measured fraction")663 #expect(ETAEstimator.remaining(fraction: 0.5, elapsed: 0) == nil)664 #expect(ETAEstimator.remaining(fraction: 0.25, elapsed: 10) == 30)665 #expect(ETAEstimator.remaining(fraction: 1, elapsed: 10) == 0)666 #expect(ETAEstimator.remaining(fraction: 1.5, elapsed: 10) == nil)667 }668669 @Test func etaAppearsOnlyOnceProgressIsMeasured() async throws {670 let bed = TestBed(steps: 20, stepDelay: 0.02)671 let stream = await bed.queue.events()672 let collector = Task { () -> (beforeProgress: Bool, etas: [Double]) in673 var etas: [Double] = []674 var nilBefore = true675 for await event in stream {676 if case .jobChanged(let job) = event {677 if job.state == .running, job.progress.fraction == nil, job.estimatedSecondsRemaining != nil { nilBefore = false }678 if job.state.isTerminal { break }679 }680 if case .jobProgress(_, _, let eta) = event, let eta { etas.append(eta) }681 }682 return (nilBefore, etas)683 }684 let id = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .yaml, destination: bed.destination()))685 let result = await collector.value686 #expect(result.beforeProgress, "no estimate before any measured fraction")687 #expect(result.etas.count >= 10 && result.etas.allSatisfy { $0 >= 0 && $0 < 30 })688 #expect(await bed.queue.job(id)?.estimatedSecondsRemaining == nil, "no estimate once finished")689 }690691 @Test func logIsBounded() async throws {692 struct Chatty: ConversionEngine {693 let id = "chatty"694 let displayName = "Chatty"695 var paths: [ConversionPath] { [ConversionPath(.txt, .markdown, engine: id, fidelity: .lossless)] }696 func inspect(_ url: URL, format: FormatID) async throws -> FileInspection { FileInspection(format: format) }697 func convert(_ request: ConversionRequest, context: ConversionContext) async throws -> ConversionOutput {698 for i in 0..<500 { context.log(.debug, "line \(i)") }699 let out = context.outputURL(for: request)700 try "x".write(to: out, atomically: true, encoding: .utf8)701 return ConversionOutput(file: out)702 }703 }704 let temp = TempDir("chatty")705 let queue = ConversionQueue(registry: FormatRegistry(engines: [Chatty()]), configuration: QueueConfiguration(maxLogLinesPerJob: 50),706 workDirectories: WorkDirectoryManager(root: temp.file("work")))707 let job = try #require(await queue.waitForJob(await queue.enqueue(ConversionJobRequest(source: temp.write("a.txt", "x"), to: .markdown))))708 #expect(job.state == .finished && job.log.count == 50 && job.log.last?.message == "Finished.")709 #expect(job.log.contains { $0.message == "line 499" } && !job.log.contains { $0.message == "line 0" })710 }711}712713@Suite struct QueueHistoryTests {714 @Test func everyEndedJobIsRecorded() async throws {715 let bed = TestBed()716 bed.temp.write("out/skip.md", "existing")717 await bed.queue.enqueue(batch: [718 ConversionJobRequest(sources: [bed.source("ok.txt", "fine")], destinationFormat: .markdown, options: ConversionOptions(["suffix": .string("!")]), destination: bed.destination()),719 ConversionJobRequest(sources: [bed.source("bad.txt")], destinationFormat: .rtf, destination: bed.destination()),720 ConversionJobRequest(sources: [bed.source("skip.txt")], destinationFormat: .markdown, destination: bed.destination(.skip)),721 ])722 await bed.queue.waitUntilIdle()723 let entries = try await bed.history.entries(HistoryQuery())724 #expect(Set(entries.map(\.status)) == ["finished", "failed", "skipped"])725 let ok = try #require(entries.first { $0.status == "finished" })726 #expect(ok.sourcePath.hasSuffix("/in/ok.txt") && ok.sourceFormat == .txt && ok.destinationFormat == .markdown && ok.engine == "mock-text")727 #expect(ok.destinationPaths.count == 1 && ok.destinationPaths[0].hasSuffix("/out/ok.md"))728 #expect(ok.options.string("suffix") == "!" && ok.bytesBefore == 4 && ok.bytesAfter == 5 && (ok.duration ?? -1) >= 0)729 #expect(ok.checks.contains { $0.name == "output.format" && $0.passed } && ok.batchID != nil && ok.error == nil)730 let bad = try #require(entries.first { $0.status == "failed" })731 #expect(bad.error?.contains("mock failure") == true && bad.destinationPaths.isEmpty)732 }733734 @Test func chainedPlansRecordBothEngines() async throws {735 let bed = TestBed()736 _ = await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .html, destination: bed.destination())))737 #expect(try await bed.history.entries().first?.engine == "mock-text+mock-html")738 }739740 @Test func recordingCanBeDisabledPerJob() async throws {741 let bed = TestBed()742 var request = ConversionJobRequest(sources: [bed.source()], destinationFormat: .markdown, destination: bed.destination())743 request.recordInHistory = false744 _ = await bed.queue.waitForJob(await bed.queue.enqueue(request))745 #expect(try await bed.history.count() == 0)746 }747748 @Test func managerEnqueuesBatchesCombinesAndAppliesPresets() async throws {749 let temp = TempDir("manager")750 let history = try ConversionHistoryStore(url: temp.file("h.sqlite"))751 let manager = ConversionJobManager(engines: [TextEngine(), HTMLEngine()], configuration: QueueConfiguration(maxConcurrentJobs: 2), history: history,752 workDirectories: WorkDirectoryManager(root: temp.file("work")), monitor: FixedMonitor())753 let out = OutputDestination(directory: temp.subdirectory("out"))754 let files = [temp.write("a.txt", "a"), temp.write("b.txt", "b")]755 let ids = await manager.convert(files, to: .markdown, destination: out)756 let merged = await manager.combine(files, to: .markdown, operation: "text.merge", destination: OutputDestination(directory: out.directory, nameTemplate: "merged"))757 let preset = ConversionPreset(name: "Text to YAML", source: .format(.txt), destinationFormat: .yaml)758 let applied = await manager.apply(preset, to: files + [temp.write("c.png", bytes: Fixtures.png)], destination: out)759 await manager.queue.waitUntilIdle()760 #expect(ids.count == 2 && applied.jobs.count == 2 && applied.skipped.map(\.lastPathComponent) == ["c.png"])761 #expect(await manager.queue.job(merged)?.state == .finished)762 #expect(temp.list("out") == ["a.md", "a.yaml", "b.md", "b.yaml", "merged.md"])763 #expect(try await history.count() == 5)764 #expect(manager.resolver.capabilities(for: .txt).isSupported && manager.registry.counts.engines == 2)765 }766}767