SPB Git

Repository Public License

QueueTests.swift 47.7 KB 767 lines swift

1// ORIX Convert — Packages/ORIXConvertCore/Tests/ConvertCoreTests/QueueTests.swift2// Description: The conversion queue end to end over the mock engine: states, chained plans, validation failures,3//              conflicts, cancellation, retry, priorities, pause, resource governance, events, ETA and history.4// Author: Simon-Pierre Boucher — contact@spboucher.ai5import Foundation6import Testing7@testable import ConvertCore89@Suite struct QueueConversionTests {10    @Test func convertsAFileEndToEnd() async throws {11        let bed = TestBed()12        let source = bed.source("note.txt", "hello world\n")13        let id = await bed.queue.enqueue(ConversionJobRequest(sources: [source], destinationFormat: .markdown, destination: bed.destination()))14        let job = try #require(await bed.queue.waitForJob(id))15        #expect(job.state == .finished)16        #expect(job.sourceFormat == .txt && job.detection?.format == .txt)17        #expect(job.plan?.steps.count == 1 && job.plan?.engines == ["mock-text"])18        #expect(job.outputs.map(\.url.lastPathComponent) == ["note.md"])19        #expect(bed.temp.read("out/note.md") == "HELLO WORLD\n")20        #expect(job.checks.allPassed && job.checks.contains { $0.name == "output.format" } && job.checks.contains { $0.name == "mock.reopen" })21        #expect(job.facts["characters"] == "12")22        #expect(job.progress.fraction == 1 && job.bytesBefore == 12 && job.bytesAfter == 12)23        #expect(job.startedAt != nil && job.finishedAt != nil && job.duration != nil)24        #expect(bed.temp.read("in/note.txt") == "hello world\n", "the source is untouched")25        #expect(bed.leftoverWorkDirectories.isEmpty, "the work directory is cleaned")26    }2728    @Test func writesNextToTheSourceByDefault() async throws {29        let bed = TestBed()30        let source = bed.source()31        let id = await bed.queue.enqueue(ConversionJobRequest(source: source, to: .markdown))32        let job = try #require(await bed.queue.waitForJob(id))33        #expect(job.outputs.first?.url.deletingLastPathComponent().lastPathComponent == "in")34    }3536    @Test func jobLogIsTimestampedAndOrdered() async throws {37        let bed = TestBed()38        let id = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .markdown, destination: bed.destination()))39        let job = try #require(await bed.queue.waitForJob(id))40        let messages = job.log.map(\.message)41        #expect(messages.first == "Queued." && messages.last == "Finished.")42        #expect(messages.contains { $0.hasPrefix("Detected txt") })43        #expect(messages.contains { $0.hasPrefix("Plan: txt → markdown") })44        #expect(messages.contains("mock converting note.txt"), "engine log lines are collected")45        #expect(messages.contains("Limitation: Text is upper-cased."))46        #expect(zip(job.log, job.log.dropFirst()).allSatisfy { $0.date <= $1.date })47    }4849    @Test func passesOptionsToTheEngine() async throws {50        let bed = TestBed()51        let request = ConversionJobRequest(sources: [bed.source("a.txt", "x")], destinationFormat: .markdown, options: ConversionOptions(["suffix": .string("!")]),52                                           destination: bed.destination())53        _ = await bed.queue.waitForJob(await bed.queue.enqueue(request))54        #expect(bed.temp.read("out/a.md") == "X!")55    }5657    @Test func executesAChainedPlanThroughAnIntermediate() async throws {58        let bed = TestBed()59        let request = ConversionJobRequest(sources: [bed.source("page.txt", "body")], destinationFormat: .html, options: ConversionOptions(["suffix": .string("?")]),60                                           destination: bed.destination())61        let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(request)))62        #expect(job.state == .finished)63        #expect(job.plan?.isChained == true && job.plan?.intermediates == [.markdown])64        #expect(bed.temp.list("out") == ["page.html"], "the intermediate never reaches the destination")65        // Step 1 ran with default options (no suffix); step 2 received the upper-cased text.66        #expect(bed.temp.read("out/page.html").contains("<pre>BODY</pre>"))67        #expect(job.checks.contains { $0.name.hasPrefix("step1.") } && job.checks.contains { $0.name.hasPrefix("step2.") })68        #expect(job.log.contains { $0.message.hasPrefix("Step 2 of 2") })69        #expect(bed.leftoverWorkDirectories.isEmpty)70    }7172    @Test func chainingCanBeRefusedPerJob() async throws {73        let bed = TestBed()74        var request = ConversionJobRequest(sources: [bed.source()], destinationFormat: .html, destination: bed.destination())75        request.allowChaining = false76        let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(request)))77        #expect(job.state == .failed && job.failure == .unsupportedPath(source: .txt, destination: .html))78    }7980    @Test func detectsTheFormatFromContentNotFromTheName() async throws {81        let bed = TestBed()82        let source = bed.source("misnamed.png", "really just text\n")83        let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(sources: [source], destinationFormat: .markdown, destination: bed.destination()))))84        #expect(job.state == .finished && job.sourceFormat == .txt)85        #expect(job.warnings.contains { $0.code == "extension-mismatch" })86    }8788    @Test func unknownFormatAndUnsupportedPathFailCleanly() async throws {89        let bed = TestBed()90        let blob = bed.temp.write("in/blob.xyz", bytes: [0xDE, 0xAD, 0xBE, 0xEF, 0, 1, 2, 3])91        let unknown = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(source: blob, to: .markdown))))92        #expect(unknown.state == .failed)93        if case .malformedInput = unknown.failure {} else { Issue.record("expected malformedInput, got \(String(describing: unknown.failure))") }94        let unsupported = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(source: bed.source(), to: .mp3))))95        #expect(unsupported.failure == .unsupportedPath(source: .txt, destination: .mp3))96    }9798    @Test func missingSourceFails() async throws {99        let bed = TestBed()100        let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(source: bed.temp.file("in/ghost.txt"), to: .markdown))))101        #expect(job.state == .failed)102        if case .unreadableInput = job.failure {} else { Issue.record("expected unreadableInput") }103    }104105    @Test func engineErrorsFailOnlyThatJob() async throws {106        let bed = TestBed()107        let ids = await bed.queue.enqueue(batch: [108            ConversionJobRequest(sources: [bed.source("a.txt")], destinationFormat: .markdown, destination: bed.destination()),109            ConversionJobRequest(sources: [bed.source("b.txt")], destinationFormat: .rtf, destination: bed.destination()),110            ConversionJobRequest(sources: [bed.source("c.txt")], destinationFormat: .markdown, destination: bed.destination()),111        ])112        await bed.queue.waitUntilIdle()113        let jobs = await bed.queue.snapshot().jobs114        #expect(jobs.map(\.state) == [.finished, .failed, .finished])115        #expect(jobs[1].failure == .malformedInput("mock failure") && jobs[1].error?.contains("mock failure") == true)116        #expect(Set(jobs.compactMap(\.request.batchID)).count == 1 && jobs.map(\.request.counter) == [1, 2, 3])117        #expect(bed.temp.list("out") == ["a.md", "c.md"])118        #expect(ids.count == 3 && bed.leftoverWorkDirectories.isEmpty)119    }120121    @Test func outputOfTheWrongFormatIsRejectedAndNeverDelivered() async throws {122        let bed = TestBed()123        let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .png, destination: bed.destination()))))124        #expect(job.state == .failed)125        if case .outputInvalid(let message) = job.failure { #expect(message.contains("png")) } else { Issue.record("expected outputInvalid") }126        #expect(job.checks.contains { $0.name == "output.format" && !$0.passed })127        #expect(bed.temp.list("out").isEmpty && bed.leftoverWorkDirectories.isEmpty)128    }129130    @Test func emptyOutputIsRejected() async throws {131        let bed = TestBed()132        let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .csv, destination: bed.destination()))))133        #expect(job.state == .failed && job.checks.contains { $0.name == "output.not-empty" && !$0.passed })134        #expect(bed.temp.list("out").isEmpty)135    }136137    @Test func engineValidationFailureBlocksDelivery() async throws {138        let bed = TestBed()139        let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .toml, destination: bed.destination()))))140        #expect(job.state == .failed && job.failure == .outputInvalid("The mock refuses its TOML output."))141        #expect(bed.temp.list("out").isEmpty)142    }143144    @Test func directoryOutputWithEscapingSymlinkIsRejected() async throws {145        let bed = TestBed()146        let request = ConversionJobRequest(sources: [bed.source()], destinationFormat: .folder, operation: "text.escape", destination: bed.destination())147        let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(request)))148        #expect(job.state == .failed && job.checks.contains { $0.name == "output.no-symlink-escape" && !$0.passed })149        #expect(bed.temp.list("out").isEmpty)150    }151152    @Test func directoryOutputIsDelivered() async throws {153        let bed = TestBed()154        let request = ConversionJobRequest(sources: [bed.source("list.txt", "a\nb\nc\n")], destinationFormat: .folder, operation: "text.explode", destination: bed.destination())155        let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(request)))156        #expect(job.state == .finished)157        #expect(bed.temp.list("out/list") == ["line-1.txt", "line-2.txt", "line-3.txt"])158        #expect(job.bytesAfter == 3)159    }160161    @Test func multipleOutputsAreAllDelivered() async throws {162        let bed = TestBed()163        let request = ConversionJobRequest(sources: [bed.source("rows.txt", "a\nb\n")], destinationFormat: .tsv, destination: bed.destination(template: "{name}_out"))164        let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(request)))165        // Several files for one job: ONE folder named like the output, holding them (never loose files).166        #expect(job.state == .finished && bed.temp.list("out") == ["rows_out"] && bed.temp.list("out/rows_out") == ["rows_out-1.tsv", "rows_out-2.tsv"])167        #expect(job.outputs.map(\.url.lastPathComponent) == ["rows_out"] && job.facts["outputFiles"] == "2")168        // Asked not to group (what workflows do): the loose files.169        var loose = ConversionJobRequest(sources: [bed.source("rows.txt", "a\nb\n")], destinationFormat: .tsv, destination: bed.destination(template: "{name}_loose"))170        loose.groupMultipleOutputs = false171        let second = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(loose)))172        #expect(second.outputs.map(\.url.lastPathComponent) == ["rows_loose-1.tsv", "rows_loose-2.tsv"])173    }174175    @Test func multiSourceOperation() async throws {176        let bed = TestBed()177        let sources = [bed.source("1.txt", "one"), bed.source("2.txt", "two"), bed.source("3.txt", "three")]178        let merge = ConversionJobRequest(sources: sources, destinationFormat: .markdown, operation: "text.merge", destination: bed.destination())179        let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(merge)))180        #expect(job.state == .finished && bed.temp.read("out/1.md") == "one\ntwo\nthree" && job.bytesBefore == 11)181        // A single-source path refuses several inputs instead of ignoring the extra ones.182        let wrong = ConversionJobRequest(sources: sources, destinationFormat: .markdown, destination: bed.destination())183        let failed = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(wrong)))184        if case .invalidOption = failed.failure {} else { Issue.record("expected invalidOption") }185    }186187    @Test func warningsFromEngineAndPreflightAreCollected() async throws {188        let bed = TestBed()189        let request = ConversionJobRequest(sources: [bed.source()], destinationFormat: .latex, options: ConversionOptions(["warn": .bool(true)]), destination: bed.destination())190        let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(request)))191        #expect(job.state == .finished)192        #expect(job.warnings.map(\.code).sorted() == ["preflight-test", "styles-lost"])193        #expect(job.warnings.contains { $0.severity == .dataLoss })194        #expect(job.log.contains { $0.level == .warning && $0.message == "Styles were dropped." })195    }196197    @Test func nameTemplateUsesMeasuredOutputDimensions() async throws {198        let bed = TestBed()199        let request = ConversionJobRequest(sources: [bed.source("img.txt", "abc")], destinationFormat: .latex, destination: bed.destination(template: "{name}_{width}x{height}"), counter: 4)200        let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(request)))201        // The source inspection says 3x1; the engine measured 640x480 on its output.202        #expect(job.outputs.map(\.url.lastPathComponent) == ["img_640x480.tex"])203        let counter = ConversionJobRequest(sources: [bed.source("img.txt", "abc")], destinationFormat: .markdown, destination: bed.destination(template: "{counter:2}-{name}-{ext}"), counter: 4)204        let second = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(counter)))205        #expect(second.outputs.map(\.url.lastPathComponent) == ["04-img-txt.md"])206    }207208    @Test func forcedEngineIsRespected() async throws {209        var other = TextEngine()210        other.id = "mock-b"211        let bed = TestBed(extraEngines: [other])212        var request = ConversionJobRequest(sources: [bed.source()], destinationFormat: .markdown, destination: bed.destination())213        request.engineID = "mock-b"214        let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(request)))215        #expect(job.state == .finished && job.plan?.engines == ["mock-b"])216    }217}218219@Suite struct QueueConflictTests {220    @Test func renamePolicyKeepsBoth() async throws {221        let bed = TestBed()222        bed.temp.write("out/note.md", "existing")223        let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .markdown, destination: bed.destination(.rename)))))224        #expect(job.outputs.map(\.outcome) == [.renamed] && bed.temp.list("out") == ["note 2.md", "note.md"])225        #expect(bed.temp.read("out/note.md") == "existing")226    }227228    @Test func skipPolicySkipsBeforeConverting() async throws {229        let bed = TestBed()230        bed.temp.write("out/note.md", "existing")231        let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .markdown, destination: bed.destination(.skip)))))232        #expect(job.state == .skipped && job.outputs.isEmpty && bed.temp.read("out/note.md") == "existing")233        #expect(bed.probe.order.isEmpty, "nothing was converted")234    }235236    @Test func replacePolicyReplaces() async throws {237        let bed = TestBed()238        bed.temp.write("out/note.md", "existing")239        let job = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("note.txt", "new")], destinationFormat: .markdown, destination: bed.destination(.replace)))))240        #expect(job.outputs.map(\.outcome) == [.replaced] && bed.temp.read("out/note.md") == "NEW")241    }242243    @Test func askUserSurfacesADecisionBeforeConverting() async throws {244        let bed = TestBed()245        bed.temp.write("out/note.md", "existing")246        let id = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("note.txt", "new")], destinationFormat: .markdown, destination: bed.destination(.askUser)))247        let waiting = try #require(await bed.queue.waitForJob(id))248        #expect(waiting.state == .awaitingDecision)249        #expect(waiting.conflict?.existing.lastPathComponent == "note.md" && waiting.conflict?.conversionDone == false && waiting.conflict?.isSource == false)250        #expect(bed.probe.order.isEmpty && bed.temp.read("out/note.md") == "existing")251        #expect(await bed.queue.resolveConflict(id, policy: .askUser) == false, "asking again is not a decision")252        #expect(await bed.queue.resolveConflict(id, policy: .rename))253        let done = try #require(await bed.queue.waitForJob(id))254        #expect(done.state == .finished && done.conflict == nil && bed.temp.read("out/note 2.md") == "NEW")255    }256257    @Test func askUserDecisionCanBeSkipOrReplace() async throws {258        let bed = TestBed()259        bed.temp.write("out/a.md", "existing")260        bed.temp.write("out/b.md", "existing")261        let a = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("a.txt", "new")], destinationFormat: .markdown, destination: bed.destination(.askUser)))262        let b = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("b.txt", "new")], destinationFormat: .markdown, destination: bed.destination(.askUser)))263        _ = await bed.queue.waitForJob(a)264        _ = await bed.queue.waitForJob(b)265        #expect(await bed.queue.resolveConflict(a, policy: .skip))266        #expect(await bed.queue.resolveConflict(b, policy: .replace))267        #expect(await bed.queue.waitForJob(a)?.state == .skipped)268        #expect(await bed.queue.waitForJob(b)?.state == .finished)269        #expect(bed.temp.read("out/a.md") == "existing" && bed.temp.read("out/b.md") == "NEW")270    }271272    @Test func conflictAppearingDuringConversionHoldsTheOutput() async throws {273        let bed = TestBed(steps: 20, stepDelay: 0.02)274        let id = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("slow.txt")], destinationFormat: .yaml, destination: bed.destination(.askUser)))275        #expect(await eventually { await bed.queue.job(id)?.state == .running })276        bed.temp.write("out/slow.yaml", "appeared: meanwhile\n")277        let waiting = try #require(await bed.queue.waitForJob(id))278        #expect(waiting.state == .awaitingDecision && waiting.conflict?.conversionDone == true)279        #expect(bed.leftoverWorkDirectories.count == 1, "the converted file is held in its work directory")280        #expect(await bed.queue.snapshot().status.active == 0, "a waiting job holds no slot")281        #expect(await bed.queue.resolveConflict(id, policy: .rename))282        let done = try #require(await bed.queue.waitForJob(id))283        #expect(done.state == .finished && bed.temp.exists("out/slow 2.yaml") && bed.temp.read("out/slow.yaml") == "appeared: meanwhile\n")284        #expect(bed.leftoverWorkDirectories.isEmpty && bed.probe.order.count == 1, "resolving does not convert again")285    }286287    @Test func cancellingAHeldJobCleansItsWorkDirectory() async throws {288        let bed = TestBed(steps: 10, stepDelay: 0.02)289        let id = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("slow.txt")], destinationFormat: .yaml, destination: bed.destination(.askUser)))290        #expect(await eventually { await bed.queue.job(id)?.state == .running })291        bed.temp.write("out/slow.yaml", "x: 1\n")292        #expect(await bed.queue.waitForJob(id)?.state == .awaitingDecision)293        #expect(await bed.queue.cancel(id))294        #expect(await bed.queue.waitForJob(id)?.state == .cancelled)295        #expect(bed.leftoverWorkDirectories.isEmpty)296    }297298    @Test func neverOverwritesTheSourceInPlace() async throws {299        let bed = TestBed()300        // txt → markdown named ".txt"?  Use a markdown-named text source so the output name collides with the source.301        let source = bed.temp.write("in/doc.md", "plain words only\n")302        var request = ConversionJobRequest(sources: [source], sourceFormat: .txt, destinationFormat: .markdown, destination: OutputDestination(conflictPolicy: .replace))303        let refused = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(request)))304        #expect(refused.state == .failed)305        if case .destinationExists = refused.failure {} else { Issue.record("expected destinationExists") }306        #expect(bed.temp.read("in/doc.md") == "plain words only\n" && bed.probe.order.isEmpty)307308        request.destination = OutputDestination(conflictPolicy: .rename)309        let renamed = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(request)))310        #expect(renamed.outputs.map(\.url.lastPathComponent) == ["doc 2.md"] && bed.temp.read("in/doc.md") == "plain words only\n")311312        request.destination = OutputDestination(conflictPolicy: .askUser)313        let asked = await bed.queue.enqueue(request)314        #expect(await bed.queue.waitForJob(asked)?.conflict?.isSource == true)315        #expect(await bed.queue.resolveConflict(asked, policy: .replace), "accepted, the job is converted…")316        let stillRefused = try #require(await bed.queue.waitForJob(asked))317        #expect(stillRefused.state == .failed, "…but replacing the source needs the explicit flag")318        #expect(bed.temp.read("in/doc.md") == "plain words only\n")319320        request.destination = OutputDestination(conflictPolicy: .replace, allowReplacingSource: true)321        let replaced = try #require(await bed.queue.waitForJob(await bed.queue.enqueue(request)))322        #expect(replaced.state == .finished && bed.temp.read("in/doc.md") == "PLAIN WORDS ONLY\n")323    }324}325326@Suite struct QueueControlTests {327    @Test func cancelsARunningJob() async throws {328        let bed = TestBed(steps: 500, stepDelay: 0.02)329        let id = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .yaml, destination: bed.destination()))330        #expect(await eventually { await bed.queue.job(id)?.state == .running })331        let started = Date()332        #expect(await bed.queue.cancel(id))333        #expect(await bed.queue.job(id)?.cancelRequested == true)334        let job = try #require(await bed.queue.waitForJob(id))335        #expect(job.state == .cancelled && Date().timeIntervalSince(started) < 3, "the engine stopped promptly, long before its 10 s run")336        #expect(bed.temp.list("out").isEmpty && bed.leftoverWorkDirectories.isEmpty)337        #expect(await bed.queue.cancel(id) == false, "already settled")338    }339340    @Test func cancelsAHelperProcessThroughProcessRunner() async throws {341        struct SleepEngine: ConversionEngine {342            let id = "sleep"343            let displayName = "Sleep"344            var paths: [ConversionPath] { [ConversionPath(.txt, .yaml, engine: id, fidelity: .lossless)] }345            func inspect(_ url: URL, format: FormatID) async throws -> FileInspection { FileInspection(format: format) }346            func convert(_ request: ConversionRequest, context: ConversionContext) async throws -> ConversionOutput {347                try await ProcessRunner.runChecked(URL(fileURLWithPath: "/bin/sleep"), arguments: ["30"])348                return ConversionOutput(files: [])349            }350        }351        let temp = TempDir("sleep")352        let queue = ConversionQueue(registry: FormatRegistry(engines: [SleepEngine()]), workDirectories: WorkDirectoryManager(root: temp.file("work")))353        let id = await queue.enqueue(ConversionJobRequest(source: temp.write("a.txt", "x"), to: .yaml))354        #expect(await eventually { await queue.job(id)?.state == .running })355        try await Task.sleep(nanoseconds: 200_000_000)356        let started = Date()357        await queue.cancel(id)358        #expect(await queue.waitForJob(id)?.state == .cancelled)359        #expect(Date().timeIntervalSince(started) < 5)360    }361362    @Test func cancelsQueuedJobsImmediatelyAndCancelAll() async throws {363        let bed = TestBed(maxConcurrent: 1, steps: 500)364        let ids = await bed.queue.enqueue(batch: (0..<4).map { ConversionJobRequest(sources: [bed.source("f\($0).txt")], destinationFormat: .yaml, destination: bed.destination()) })365        #expect(await eventually { await bed.queue.job(ids[0])?.state == .running })366        #expect(await bed.queue.cancel(ids[3]))367        #expect(await bed.queue.job(ids[3])?.state == .cancelled)368        await bed.queue.cancelAll()369        await bed.queue.waitUntilIdle()370        #expect(await bed.queue.snapshot().jobs.map(\.state) == [.cancelled, .cancelled, .cancelled, .cancelled])371        #expect(bed.probe.order == ["f0.txt"], "the waiting jobs never started")372    }373374    @Test func retryRunsAFailedOrCancelledJobAgain() async throws {375        let bed = TestBed(steps: 500)376        let id = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .yaml, destination: bed.destination()))377        #expect(await eventually { await bed.queue.job(id)?.state == .running })378        #expect(await bed.queue.retry(id) == false, "a running job cannot be retried")379        await bed.queue.cancel(id)380        #expect(await bed.queue.waitForJob(id)?.state == .cancelled)381382        let failing = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .rtf, destination: bed.destination()))383        #expect(await bed.queue.waitForJob(failing)?.state == .failed)384        #expect(await bed.queue.retry(failing))385        let again = try #require(await bed.queue.waitForJob(failing))386        #expect(again.state == .failed && again.attempt == 2 && again.log.contains { $0.message == "Retry (attempt 2)." })387        #expect(try await bed.history.count() == 3, "each ended attempt is recorded")388    }389390    @Test func retryAfterFixingTheCauseSucceeds() async throws {391        let bed = TestBed()392        let source = bed.temp.file("in/late.txt")393        let id = await bed.queue.enqueue(ConversionJobRequest(sources: [source], sourceFormat: .txt, destinationFormat: .markdown, destination: bed.destination()))394        #expect(await bed.queue.waitForJob(id)?.state == .failed)395        bed.temp.write("in/late.txt", "now here")396        #expect(await bed.queue.retry(id))397        let job = try #require(await bed.queue.waitForJob(id))398        #expect(job.state == .finished && job.failure == nil && job.attempt == 2)399    }400401    @Test func queuePauseHoldsNewStartsAndLetsRunningJobsFinish() async throws {402        let bed = TestBed(maxConcurrent: 1, steps: 10)403        let first = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("a.txt")], destinationFormat: .yaml, destination: bed.destination()))404        let second = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("b.txt")], destinationFormat: .markdown, destination: bed.destination()))405        #expect(await eventually { await bed.queue.job(first)?.state == .running })406        await bed.queue.pause()407        #expect(await bed.queue.isPaused)408        #expect(await bed.queue.waitForJob(first)?.state == .finished, "a running job is not interrupted by pausing the queue")409        await bed.queue.waitUntilIdle()410        #expect(await bed.queue.job(second)?.state == .queued)411        await bed.queue.resume()412        #expect(await bed.queue.waitForJob(second)?.state == .finished)413    }414415    @Test func jobPauseAppliesBeforeStartOnly() async throws {416        let bed = TestBed(maxConcurrent: 1, steps: 15)417        let running = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("a.txt")], destinationFormat: .yaml, destination: bed.destination()))418        let waiting = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("b.txt")], destinationFormat: .markdown, destination: bed.destination()))419        #expect(await eventually { await bed.queue.job(running)?.state == .running })420        #expect(await bed.queue.pauseJob(running) == false, "no cooperative suspension: a running job is not paused")421        #expect(await bed.queue.pauseJob(waiting))422        #expect(await bed.queue.waitForJob(running)?.state == .finished)423        await bed.queue.waitUntilIdle()424        #expect(await bed.queue.job(waiting)?.state == .paused && bed.probe.order == ["a.txt"])425        #expect(await bed.queue.resumeJob(waiting))426        #expect(await bed.queue.waitForJob(waiting)?.state == .finished)427        #expect(await bed.queue.resumeJob(waiting) == false)428    }429430    @Test func higherPriorityStartsFirst() async throws {431        let bed = TestBed(maxConcurrent: 1)432        await bed.queue.pause()433        func request(_ name: String, _ priority: JobPriority) -> ConversionJobRequest {434            ConversionJobRequest(sources: [bed.source(name)], destinationFormat: .markdown, destination: bed.destination(), priority: priority)435        }436        await bed.queue.enqueue(request("low.txt", .low))437        await bed.queue.enqueue(request("normal1.txt", .normal))438        await bed.queue.enqueue(request("high.txt", .high))439        await bed.queue.enqueue(request("normal2.txt", .normal))440        await bed.queue.resume()441        await bed.queue.waitUntilIdle()442        #expect(bed.probe.order == ["high.txt", "normal1.txt", "normal2.txt", "low.txt"])443    }444445    @Test func prioritiesAndOrderCanBeChangedWhileWaiting() async throws {446        let bed = TestBed(maxConcurrent: 1)447        await bed.queue.pause()448        let ids = await bed.queue.enqueue(batch: ["a", "b", "c", "d"].map { ConversionJobRequest(sources: [bed.source($0 + ".txt")], destinationFormat: .markdown, destination: bed.destination()) })449        #expect(await bed.queue.setPriority(.high, for: ids[2]))450        #expect(await bed.queue.move(ids[3], to: 0))451        #expect(await bed.queue.snapshot().jobs.map(\.id) == [ids[3], ids[0], ids[1], ids[2]])452        await bed.queue.resume()453        await bed.queue.waitUntilIdle()454        #expect(bed.probe.order == ["c.txt", "d.txt", "a.txt", "b.txt"])455        #expect(await bed.queue.setPriority(.low, for: ids[0]) == false, "a finished job has no priority to change")456    }457458    @Test func removeAndClearSettled() async throws {459        let bed = TestBed(maxConcurrent: 1, steps: 500)460        let running = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("a.txt")], destinationFormat: .yaml, destination: bed.destination()))461        let waiting = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source("b.txt")], destinationFormat: .markdown, destination: bed.destination()))462        #expect(await eventually { await bed.queue.job(running)?.state == .running })463        #expect(await bed.queue.remove(running) == false, "running jobs must be cancelled first")464        #expect(await bed.queue.remove(waiting))465        #expect(await bed.queue.job(waiting) == nil)466        await bed.queue.cancel(running)467        _ = await bed.queue.waitForJob(running)468        #expect(await bed.queue.clearSettled() == 1)469        #expect(await bed.queue.snapshot().jobs.isEmpty)470        #expect(await bed.queue.waitForJob(UUID()) == nil)471    }472}473474@Suite struct QueueGovernanceTests {475    private func runBatch(_ bed: TestBed, count: Int, to format: FormatID = .yaml) async {476        await bed.queue.enqueue(batch: (0..<count).map { ConversionJobRequest(sources: [bed.source("f\($0).txt")], destinationFormat: format, destination: bed.destination()) })477        await bed.queue.waitUntilIdle()478    }479480    @Test func respectsTheConcurrencyLimit() async throws {481        let bed = TestBed(maxConcurrent: 3, steps: 5)482        await runBatch(bed, count: 10)483        #expect(bed.probe.maximum == 3)484        #expect(await bed.queue.snapshot().jobs.allSatisfy { $0.state == .finished })485    }486487    @Test func heavyJobsHaveFewerSlotsAndDoNotBlockLightOnes() async throws {488        let bed = TestBed(maxConcurrent: 4, maxHeavy: 1, steps: 8, heavyEngines: ["mock-text"])489        var heavy = (0..<3).map { ConversionJobRequest(sources: [bed.source("h\($0).txt")], destinationFormat: .yaml, destination: bed.destination()) }490        for index in heavy.indices { heavy[index].sourceFormat = .txt }491        let ids = await bed.queue.enqueue(batch: heavy)492        #expect(await bed.queue.job(ids[0])?.isHeavy == true)493        await bed.queue.waitUntilIdle()494        #expect(bed.probe.maximum == 1, "one heavy slot")495    }496497    @Test func heavyClassification() async throws {498        let bed = TestBed(heavyEngines: ["mock-text"])499        let light = FormatRegistry(engines: [TextEngine(), HTMLEngine()])500        let lightQueue = ConversionQueue(registry: light, configuration: QueueConfiguration(maxConcurrentJobs: 2, heavyFileSize: 10),501                                         workDirectories: bed.workDirectories, monitor: bed.monitor)502        await lightQueue.pause()503        let small = await lightQueue.enqueue(ConversionJobRequest(source: bed.source("s.txt", "tiny"), to: .markdown))504        let large = await lightQueue.enqueue(ConversionJobRequest(source: bed.source("l.txt", String(repeating: "x", count: 64)), to: .markdown))505        let video = await lightQueue.enqueue(ConversionJobRequest(source: bed.source("v.txt"), to: .mp4))506        #expect(await lightQueue.job(small)?.isHeavy == false)507        #expect(await lightQueue.job(large)?.isHeavy == true, "above the size threshold")508        #expect(await lightQueue.job(video)?.isHeavy == true, "video destination")509        await bed.queue.pause()510        let forced = await bed.queue.enqueue(ConversionJobRequest(source: bed.source("e.txt"), to: .markdown))511        #expect(await bed.queue.job(forced)?.isHeavy == true, "planned on a heavy engine")512    }513514    @Test func limitsFollowThermalAndPowerState() {515        let config = QueueConfiguration(maxConcurrentJobs: 8, maxHeavyJobs: 2, reduceInLowPowerMode: true, reduceOnBattery: true)516        #expect(config.limits(for: ResourceState()) == ConcurrencyLimits(total: 8, heavy: 2, reason: nil))517        #expect(config.limits(for: ResourceState(thermal: .fair)).total == 8)518        let serious = config.limits(for: ResourceState(thermal: .serious))519        #expect(serious.total == 4 && serious.heavy == 1 && serious.reason != nil)520        let critical = config.limits(for: ResourceState(thermal: .critical))521        #expect(critical.total == 0 && critical.heavy == 0)522        let lowPower = config.limits(for: ResourceState(lowPowerMode: true))523        #expect(lowPower.total == 4 && lowPower.heavy == 1 && lowPower.reason?.contains("Low Power Mode") == true)524        #expect(config.limits(for: ResourceState(onBattery: true)).total == 4)525        let optedOut = QueueConfiguration(maxConcurrentJobs: 8, reduceInLowPowerMode: false, reduceOnBattery: false)526        #expect(optedOut.limits(for: ResourceState(lowPowerMode: true, onBattery: true)).total == 8)527        #expect(QueueConfiguration(maxConcurrentJobs: 1).limits(for: ResourceState(thermal: .serious, lowPowerMode: true)).total == 1)528        #expect(QueueConfiguration(maxConcurrentJobs: 2, maxHeavyJobs: 9).limits(for: ResourceState()).heavy == 2, "heavy never exceeds total")529    }530531    @Test func defaultConcurrencyComesFromPerformanceCores() {532        let cores = QueueConfiguration.defaultConcurrency()533        #expect((1...8).contains(cores))534        #expect(QueueConfiguration().maxConcurrentJobs == cores && QueueConfiguration().maxHeavyJobs >= 1)535        #expect(QueueConfiguration(maxConcurrentJobs: 0).maxConcurrentJobs == 1)536    }537538    @Test func seriousThermalStateHalvesConcurrency() async throws {539        let bed = TestBed(maxConcurrent: 4, state: ResourceState(thermal: .serious), steps: 5)540        await runBatch(bed, count: 8)541        #expect(bed.probe.maximum == 2)542        #expect(await bed.queue.snapshot().status.limits.reason?.contains("thermal") == true)543    }544545    @Test func criticalThermalStatePausesNewStartsUntilItCools() async throws {546        let bed = TestBed(maxConcurrent: 4, state: ResourceState(thermal: .critical))547        let id = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .markdown, destination: bed.destination()))548        try await Task.sleep(nanoseconds: 300_000_000)549        #expect(await bed.queue.job(id)?.state == .queued && bed.probe.order.isEmpty)550        #expect(await bed.queue.snapshot().status.limits.total == 0)551        bed.monitor.box.mutate { $0.thermal = .nominal }552        await bed.queue.resourceStateChanged()553        #expect(await bed.queue.waitForJob(id)?.state == .finished)554    }555556    @Test func lowPowerModeReducesConcurrencyWhenEnabled() async throws {557        let bed = TestBed(maxConcurrent: 4, state: ResourceState(lowPowerMode: true), steps: 5)558        await runBatch(bed, count: 8)559        #expect(bed.probe.maximum == 2)560    }561562    @Test func configurationCanBeChangedLive() async throws {563        let bed = TestBed(maxConcurrent: 1, steps: 5)564        await bed.queue.pause()565        await bed.queue.enqueue(batch: (0..<6).map { ConversionJobRequest(sources: [bed.source("f\($0).txt")], destinationFormat: .yaml, destination: bed.destination()) })566        await bed.queue.updateConfiguration(QueueConfiguration(maxConcurrentJobs: 3))567        #expect(await bed.queue.currentConfiguration.maxConcurrentJobs == 3)568        await bed.queue.resume()569        await bed.queue.waitUntilIdle()570        #expect(bed.probe.maximum == 3)571    }572573    @Test func systemMonitorReadsRealState() {574        let state = SystemResourceMonitor().currentState()575        #expect(state.thermal >= .nominal && state.thermal <= .critical)576        _ = SystemResourceMonitor.isOnBattery()577        #expect(ThermalLevel(.serious) == .serious && ThermalLevel(.nominal) < ThermalLevel(.critical))578    }579}580581@Suite struct QueueObservationTests {582    @Test func eventsDescribeTheLifeOfAJob() async throws {583        let bed = TestBed(steps: 5)584        let stream = await bed.queue.events()585        let collector = Task { () -> [QueueEvent] in586            var events: [QueueEvent] = []587            for await event in stream {588                events.append(event)589                if case .jobChanged(let job) = event, job.state.isTerminal { break }590            }591            return events592        }593        let id = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .yaml, destination: bed.destination()))594        let events = await collector.value595        var states: [JobState] = []596        var fractions: [Double] = []597        var sawAdded = false, sawLog = false598        for event in events {599            switch event {600            case .jobAdded(let job): sawAdded = job.id == id601            case .jobChanged(let job): if states.last != job.state { states.append(job.state) }602            case .jobProgress(_, let progress, _): if let f = progress.fraction { fractions.append(f) }603            case .jobLog: sawLog = true604            default: break605            }606        }607        #expect(sawAdded && sawLog)608        #expect(states == [.preparing, .running, .validating, .finished])609        #expect(fractions == fractions.sorted() && fractions.last == 1 && fractions.count == 5, "measured progress, in order")610    }611612    @Test func severalSubscribersAndUnsubscription() async throws {613        let bed = TestBed()614        let first = await bed.queue.events()615        let second = await bed.queue.events()616        let counter = Task { () -> Int in617            var n = 0618            for await event in second { if case .jobAdded = event { n += 1; break } }619            return n620        }621        let dropped = Task { for await _ in first { break } }622        await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .markdown, destination: bed.destination()))623        #expect(await counter.value == 1)624        await dropped.value625        await bed.queue.waitUntilIdle()626    }627628    @Test func idleEventAndSnapshotStatus() async throws {629        let bed = TestBed(maxConcurrent: 2, steps: 5)630        let stream = await bed.queue.events()631        let idle = Task { () -> Bool in632            for await event in stream { if case .idle = event { return true } }633            return false634        }635        await bed.queue.enqueue(batch: (0..<3).map { ConversionJobRequest(sources: [bed.source("f\($0).txt")], destinationFormat: .yaml, destination: bed.destination()) })636        #expect(await idle.value)637        let snapshot = await bed.queue.snapshot()638        #expect(snapshot.status.active == 0 && snapshot.status.queued == 0 && snapshot.status.settled == 3 && !snapshot.status.isPaused)639        #expect(snapshot.status.limits.total == 2)640    }641642    @Test func progressOfAChainedPlanSpansAllSteps() async throws {643        let bed = TestBed()644        let stream = await bed.queue.events()645        let collector = Task { () -> [JobProgress] in646            var all: [JobProgress] = []647            for await event in stream {648                if case .jobProgress(_, let progress, _) = event { all.append(progress) }649                if case .jobChanged(let job) = event, job.state.isTerminal { break }650            }651            return all652        }653        await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .html, destination: bed.destination()))654        let progress = await collector.value655        // Step 1 (txt → markdown) reports 1.0 of step 1 of 2 → 0.5 overall; step 2 reports nothing.656        #expect(progress.map(\.fraction) == [0.5] && progress.first?.stepCount == 2 && progress.first?.stepIndex == 0)657    }658659    @Test func etaNeedsMeasuredProgress() {660        #expect(ETAEstimator.remaining(fraction: nil, elapsed: 10) == nil)661        #expect(ETAEstimator.remaining(fraction: 0, elapsed: 10) == nil)662        #expect(ETAEstimator.remaining(fraction: 0.001, elapsed: 10) == nil, "below the minimum measured fraction")663        #expect(ETAEstimator.remaining(fraction: 0.5, elapsed: 0) == nil)664        #expect(ETAEstimator.remaining(fraction: 0.25, elapsed: 10) == 30)665        #expect(ETAEstimator.remaining(fraction: 1, elapsed: 10) == 0)666        #expect(ETAEstimator.remaining(fraction: 1.5, elapsed: 10) == nil)667    }668669    @Test func etaAppearsOnlyOnceProgressIsMeasured() async throws {670        let bed = TestBed(steps: 20, stepDelay: 0.02)671        let stream = await bed.queue.events()672        let collector = Task { () -> (beforeProgress: Bool, etas: [Double]) in673            var etas: [Double] = []674            var nilBefore = true675            for await event in stream {676                if case .jobChanged(let job) = event {677                    if job.state == .running, job.progress.fraction == nil, job.estimatedSecondsRemaining != nil { nilBefore = false }678                    if job.state.isTerminal { break }679                }680                if case .jobProgress(_, _, let eta) = event, let eta { etas.append(eta) }681            }682            return (nilBefore, etas)683        }684        let id = await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .yaml, destination: bed.destination()))685        let result = await collector.value686        #expect(result.beforeProgress, "no estimate before any measured fraction")687        #expect(result.etas.count >= 10 && result.etas.allSatisfy { $0 >= 0 && $0 < 30 })688        #expect(await bed.queue.job(id)?.estimatedSecondsRemaining == nil, "no estimate once finished")689    }690691    @Test func logIsBounded() async throws {692        struct Chatty: ConversionEngine {693            let id = "chatty"694            let displayName = "Chatty"695            var paths: [ConversionPath] { [ConversionPath(.txt, .markdown, engine: id, fidelity: .lossless)] }696            func inspect(_ url: URL, format: FormatID) async throws -> FileInspection { FileInspection(format: format) }697            func convert(_ request: ConversionRequest, context: ConversionContext) async throws -> ConversionOutput {698                for i in 0..<500 { context.log(.debug, "line \(i)") }699                let out = context.outputURL(for: request)700                try "x".write(to: out, atomically: true, encoding: .utf8)701                return ConversionOutput(file: out)702            }703        }704        let temp = TempDir("chatty")705        let queue = ConversionQueue(registry: FormatRegistry(engines: [Chatty()]), configuration: QueueConfiguration(maxLogLinesPerJob: 50),706                                    workDirectories: WorkDirectoryManager(root: temp.file("work")))707        let job = try #require(await queue.waitForJob(await queue.enqueue(ConversionJobRequest(source: temp.write("a.txt", "x"), to: .markdown))))708        #expect(job.state == .finished && job.log.count == 50 && job.log.last?.message == "Finished.")709        #expect(job.log.contains { $0.message == "line 499" } && !job.log.contains { $0.message == "line 0" })710    }711}712713@Suite struct QueueHistoryTests {714    @Test func everyEndedJobIsRecorded() async throws {715        let bed = TestBed()716        bed.temp.write("out/skip.md", "existing")717        await bed.queue.enqueue(batch: [718            ConversionJobRequest(sources: [bed.source("ok.txt", "fine")], destinationFormat: .markdown, options: ConversionOptions(["suffix": .string("!")]), destination: bed.destination()),719            ConversionJobRequest(sources: [bed.source("bad.txt")], destinationFormat: .rtf, destination: bed.destination()),720            ConversionJobRequest(sources: [bed.source("skip.txt")], destinationFormat: .markdown, destination: bed.destination(.skip)),721        ])722        await bed.queue.waitUntilIdle()723        let entries = try await bed.history.entries(HistoryQuery())724        #expect(Set(entries.map(\.status)) == ["finished", "failed", "skipped"])725        let ok = try #require(entries.first { $0.status == "finished" })726        #expect(ok.sourcePath.hasSuffix("/in/ok.txt") && ok.sourceFormat == .txt && ok.destinationFormat == .markdown && ok.engine == "mock-text")727        #expect(ok.destinationPaths.count == 1 && ok.destinationPaths[0].hasSuffix("/out/ok.md"))728        #expect(ok.options.string("suffix") == "!" && ok.bytesBefore == 4 && ok.bytesAfter == 5 && (ok.duration ?? -1) >= 0)729        #expect(ok.checks.contains { $0.name == "output.format" && $0.passed } && ok.batchID != nil && ok.error == nil)730        let bad = try #require(entries.first { $0.status == "failed" })731        #expect(bad.error?.contains("mock failure") == true && bad.destinationPaths.isEmpty)732    }733734    @Test func chainedPlansRecordBothEngines() async throws {735        let bed = TestBed()736        _ = await bed.queue.waitForJob(await bed.queue.enqueue(ConversionJobRequest(sources: [bed.source()], destinationFormat: .html, destination: bed.destination())))737        #expect(try await bed.history.entries().first?.engine == "mock-text+mock-html")738    }739740    @Test func recordingCanBeDisabledPerJob() async throws {741        let bed = TestBed()742        var request = ConversionJobRequest(sources: [bed.source()], destinationFormat: .markdown, destination: bed.destination())743        request.recordInHistory = false744        _ = await bed.queue.waitForJob(await bed.queue.enqueue(request))745        #expect(try await bed.history.count() == 0)746    }747748    @Test func managerEnqueuesBatchesCombinesAndAppliesPresets() async throws {749        let temp = TempDir("manager")750        let history = try ConversionHistoryStore(url: temp.file("h.sqlite"))751        let manager = ConversionJobManager(engines: [TextEngine(), HTMLEngine()], configuration: QueueConfiguration(maxConcurrentJobs: 2), history: history,752                                           workDirectories: WorkDirectoryManager(root: temp.file("work")), monitor: FixedMonitor())753        let out = OutputDestination(directory: temp.subdirectory("out"))754        let files = [temp.write("a.txt", "a"), temp.write("b.txt", "b")]755        let ids = await manager.convert(files, to: .markdown, destination: out)756        let merged = await manager.combine(files, to: .markdown, operation: "text.merge", destination: OutputDestination(directory: out.directory, nameTemplate: "merged"))757        let preset = ConversionPreset(name: "Text to YAML", source: .format(.txt), destinationFormat: .yaml)758        let applied = await manager.apply(preset, to: files + [temp.write("c.png", bytes: Fixtures.png)], destination: out)759        await manager.queue.waitUntilIdle()760        #expect(ids.count == 2 && applied.jobs.count == 2 && applied.skipped.map(\.lastPathComponent) == ["c.png"])761        #expect(await manager.queue.job(merged)?.state == .finished)762        #expect(temp.list("out") == ["a.md", "a.yaml", "b.md", "b.yaml", "merged.md"])763        #expect(try await history.count() == 5)764        #expect(manager.resolver.capabilities(for: .txt).isSupported && manager.registry.counts.engines == 2)765    }766}767
Files main
Menu

Theme