SPB Git

Repository Public

WorkflowRunnerTests.swift 21.2 KB 357 lines swift

1// ORIX Docs2// File: Packages/ORIXDocsKit/Tests/DocAutomationTests/WorkflowRunnerTests.swift3// Description: Whole workflows end to end, progress, cancellation, batches with a failing input, history persistence and4//              the validation of workflows written as JSON (unknown steps, paths leaving the output folder).5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import PDFKit10import Testing11import DocModel12import DocContainer13@testable import DocAutomation1415@Suite struct WorkflowRunnerTests {16    // MARK: End to end1718    @Test func docxToStyledPDFEndToEnd() async throws {19        let folder = TempFolder()20        let source = folder.file("pandoc-toc.docx")21        try FileManager.default.copyItem(at: Sample.corpus.appendingPathComponent("pandoc-toc.docx"), to: source)22        let out = folder.subfolder("out")23        let events = Collector<WorkflowProgress>()24        let result = await WorkflowRunner(outputDirectory: out).run(BuiltinWorkflows.docxToStyledPDF, input: .file(source)) { events.add($0) }2526        #expect(result.record.status == .succeeded, "\(result.record.error?.description ?? "")")27        #expect(result.record.steps.map(\.type) == ["import", "applyTheme", "tableOfContents", "audit", "export"])28        #expect(result.record.steps.allSatisfy { $0.outcome == .succeeded })29        #expect(result.record.steps.allSatisfy { $0.duration >= 0 })30        #expect(result.record.ended >= result.record.started)31        #expect(result.record.audit != nil)32        #expect(result.document?.styles.themeName == Theme.orix.name)33        #expect(result.document?.body.contains { if case .toc = $0 { return true } else { return false } } == true)3435        let pdf = try #require(PDFDocument(url: out.appendingPathComponent("pandoc-toc.pdf")))36        #expect(pdf.pageCount >= 1)37        // The headings of the source document are in the PDF, in the contents and in the text.38        let imported = try DocumentConverter.importDocument(at: source).document39        let firstHeading = try #require(Numbering(imported).outline.first?.text)40        #expect((pdf.string ?? "").contains(firstHeading))4142        // Progress: a start and a finish per step, in order, ending at 1.43        let progress = events.values44        #expect(progress.count == 10)45        #expect(progress.map(\.stepIndex) == [0, 0, 1, 1, 2, 2, 3, 3, 4, 4])46        #expect(progress.last?.fraction == 1)47        #expect(progress.map(\.fraction) == progress.map(\.fraction).sorted())48        #expect(progress.first?.stepName == "Open the input document")49    }5051    @Test func markdownToPDFWithTemplate() async throws {52        let folder = TempFolder()53        let source = folder.file("field-notes.md")54        try Data(Sample.markdown.utf8).write(to: source)55        let result = await WorkflowRunner(outputDirectory: folder.subfolder("out")).run(BuiltinWorkflows.markdownToPDF, input: .file(source))56        #expect(result.record.status == .succeeded, "\(result.record.error?.description ?? "")")57        #expect(result.document?.pageSetup.paper == .a4)58        let pdf = try #require(PDFDocument(url: folder.subfolder("out").appendingPathComponent("field-notes.pdf")))59        let bounds = try #require(pdf.page(at: 0)?.bounds(for: .mediaBox))60        #expect(abs(bounds.width - 595.28) < 1 && abs(bounds.height - 841.89) < 1)61        #expect((pdf.string ?? "").contains("Field Notes"))62        #expect((pdf.string ?? "").contains("North"))63    }6465    @Test func cleanReviewCopy() async throws {66        let folder = TempFolder()67        var doc = DocumentModel(body: [Sample.para("Budget", style: StyleID.heading(1)), Sample.para("The total is high.")])68        doc.settings.trackChanges = true69        var editor = DocumentEditor(doc: doc, selection: Selection(anchor: Position(block: 1, offset: 13), head: Position(block: 1, offset: 17)), author: "Reviewer")70        editor.insertText("low")71        editor.selection = Selection(anchor: Position(block: 1, offset: 4), head: Position(block: 1, offset: 9))72        editor.addComment("Which total?")73        let source = try Sample.write(editor.doc, to: folder.file("draft.docx"))7475        let result = await WorkflowRunner(outputDirectory: folder.subfolder("out")).run(BuiltinWorkflows.cleanReviewCopy, input: .file(source))76        #expect(result.record.status == .succeeded, "\(result.record.error?.description ?? "")")77        let clean = try DocumentConverter.importDocument(at: folder.subfolder("out").appendingPathComponent("draft-clean.docx")).document78        #expect(!clean.hasTrackedChanges)79        #expect(clean.comments.isEmpty)80        let texts = clean.body.compactMap { $0.paragraph?.plainText }81        #expect(texts.contains("The total is low."))82    }8384    @Test func auditOnlyWritesReportsAndNoDocument() async throws {85        let folder = TempFolder()86        let source = try Sample.write(Sample.report(), to: folder.file("report.orixdoc"))87        let out = folder.subfolder("out")88        let result = await WorkflowRunner(outputDirectory: out).run(BuiltinWorkflows.auditOnly, input: .file(source))89        #expect(result.record.status == .succeeded)90        #expect(try FileManager.default.contentsOfDirectory(atPath: out.path).sorted() == ["report-audit.json", "report-audit.md"])91    }9293    @Test func builtinsAreValidAndListed() throws {94        #expect(BuiltinWorkflows.all.map(\.id) == ["docx-to-styled-pdf", "markdown-to-pdf", "clean-review-copy", "audit-only"])95        for workflow in BuiltinWorkflows.all {96            try workflow.validate()97            #expect(BuiltinWorkflows.named(workflow.id) == workflow)98            // JSON round trip.99            #expect(try Workflow.load(json: workflow.jsonData()) == workflow)100        }101        #expect(BuiltinWorkflows.named("Audit only")?.id == "audit-only")102        #expect(BuiltinWorkflows.named("nope") == nil)103    }104105    // MARK: Cancellation106107    @Test func cancellationStopsBetweenSteps() async throws {108        let folder = TempFolder()109        let steps: [WorkflowStep] = [.applyTheme(name: "Modern"), .export(format: .pdf, fileName: "a.pdf", pdf: nil), .export(format: .docx, fileName: "b.docx", pdf: nil)]110        let runner = WorkflowRunner(outputDirectory: folder.url)111        let box = Collector<Task<WorkflowResult, Never>>()112        let task = Task {113            await runner.run(Workflow(name: "Slow", steps: steps), input: .document(Sample.report(), name: "doc")) { p in114                // Cancel as soon as the first step is done.115                if p.stepIndex == 0, p.phase == .finished { box.values.first?.cancel() }116            }117        }118        box.add(task)119        let result = await task.value120        // The task may have run the first step before `box` held it: accept both, but a cancellation must leave no later output.121        if result.record.status == .cancelled {122            #expect(result.record.error?.kind == .cancelled)123            #expect(result.record.steps.last?.outcome == .notRun || result.record.steps.last?.outcome == .cancelled)124            #expect(!FileManager.default.fileExists(atPath: folder.file("b.docx").path))125        }126127        // Deterministic: a task cancelled before it starts runs no step at all.128        let cancelled = Task { () -> WorkflowResult in129            withUnsafeCurrentTask { $0?.cancel() }130            return await runner.run(Workflow(name: "Slow", steps: steps), input: .document(Sample.report(), name: "never"))131        }132        let none = await cancelled.value133        #expect(none.record.status == .cancelled)134        #expect(none.record.steps.map(\.outcome) == [.cancelled, .notRun, .notRun])135        #expect(none.record.outputs.isEmpty)136        #expect(!FileManager.default.fileExists(atPath: folder.file("never.pdf").path))137    }138139    @Test func batchCancellationStopsStartingFiles() async throws {140        let folder = TempFolder()141        let inputs = try (0..<12).map { try Sample.write(Sample.report(title: "Report \($0)"), to: folder.file("r\($0).orixdoc")) }142        let workflow = Workflow(name: "PDF", steps: [.importDocument(path: nil), .export(format: .pdf, fileName: nil, pdf: nil)])143        let runner = WorkflowRunner(outputDirectory: folder.subfolder("out"))144        let box = Collector<Task<BatchResult, Never>>()145        let task = Task {146            await runner.runBatch(workflow, inputs: inputs, maxConcurrent: 1) { p in147                if p.finished != nil, p.completedFiles == 2 { box.values.first?.cancel() }148            }149        }150        box.add(task)151        let result = await task.value152        #expect(result.runs.count == 12)153        #expect(result.runs.map(\.input) == inputs.map(\.path))154        #expect(result.succeeded.count >= 2)155        #expect(result.cancelled.count >= 8)156        #expect(result.succeeded.count + result.cancelled.count + result.failed.count == 12)157        let written = try FileManager.default.contentsOfDirectory(atPath: folder.subfolder("out").path).count158        #expect(written == result.succeeded.count)159    }160161    // MARK: Batch162163    @Test func batchContinuesAfterAFailingInput() async throws {164        let folder = TempFolder()165        var inputs = try (0..<5).map { try Sample.write(Sample.report(title: "Report \($0)"), to: folder.file("report-\($0).docx")) }166        let broken = folder.file("broken.docx")167        try Data("this is not a zip archive".utf8).write(to: broken)168        inputs.insert(broken, at: 2)169        let out = folder.subfolder("out")170        let events = Collector<BatchProgress>()171        let history = WorkflowHistoryStore(url: folder.file("history.json"))172173        let result = await WorkflowRunner(outputDirectory: out, history: history).runBatch(BuiltinWorkflows.docxToStyledPDF, inputs: inputs, maxConcurrent: 3) { events.add($0) }174175        #expect(result.runs.count == 6)176        #expect(result.runs.map(\.input) == inputs.map(\.path))   // input order, whatever the completion order177        #expect(result.succeeded.count == 5)178        #expect(result.failed.count == 1)179        let failure = try #require(result.failed.first)180        #expect(failure.inputName == "broken.docx")181        #expect(failure.error?.kind == .importFailed)182        #expect(failure.error?.stepIndex == 0)183        #expect(failure.error?.description.contains("broken.docx") == true)184        #expect(failure.steps[0].outcome == .failed && failure.steps[1].outcome == .notRun)185        #expect(try FileManager.default.contentsOfDirectory(atPath: out.path).sorted() == (0..<5).map { "report-\($0).pdf" })186        for i in 0..<5 { #expect(PDFDocument(url: out.appendingPathComponent("report-\(i).pdf"))?.string?.contains("Report \(i)") == true) }187188        let finished = events.values.filter { $0.finished != nil }189        #expect(finished.count == 6)190        #expect(finished.map(\.completedFiles).sorted() == [1, 2, 3, 4, 5, 6])191        #expect(events.values.contains { $0.step != nil })192        #expect(history.load().count == 6)193    }194195    @Test func concurrencyIsBounded() async throws {196        let folder = TempFolder()197        let inputs = try (0..<8).map { try Sample.write(Sample.report(), to: folder.file("d\($0).orixdoc")) }198        let workflow = Workflow(name: "PDF", steps: [.importDocument(path: nil), .export(format: .pdf, fileName: nil, pdf: nil)])199        final class Gauge: @unchecked Sendable {200            let lock = NSLock(); var running = Set<Int>(); var peak = 0201            func update(_ p: BatchProgress) {202                lock.lock(); defer { lock.unlock() }203                if p.finished != nil { running.remove(p.fileIndex) } else { running.insert(p.fileIndex) }204                peak = max(peak, running.count)205            }206        }207        let gauge = Gauge()208        let result = await WorkflowRunner(outputDirectory: folder.subfolder("out")).runBatch(workflow, inputs: inputs, maxConcurrent: 2) { gauge.update($0) }209        #expect(result.succeeded.count == 8)210        #expect(gauge.peak >= 1 && gauge.peak <= 2)211    }212213    // MARK: History214215    @Test func historyPersistsNewestFirstAndIsBounded() async throws {216        let folder = TempFolder()217        let url = folder.subfolder("support").appendingPathComponent("history.json")218        let store = WorkflowHistoryStore(url: url, limit: 3)219        #expect(store.load().isEmpty)220        let runner = WorkflowRunner(outputDirectory: folder.subfolder("out"), existingFiles: .overwrite, history: store)221        for i in 0..<5 {222            _ = await runner.run(Workflow(name: "Run \(i)", steps: [.export(format: .txt, fileName: nil, pdf: nil)]), input: .document(Sample.report(), name: "doc"))223        }224        // A new store on the same file sees what the first one wrote.225        let reopened = WorkflowHistoryStore(url: url, limit: 3).load()226        #expect(reopened.map(\.workflowName) == ["Run 4", "Run 3", "Run 2"])227        let latest = try #require(reopened.first)228        #expect(latest.status == .succeeded)229        #expect(latest.steps.count == 1 && latest.steps[0].outcome == .succeeded)230        #expect(latest.outputs.count == 1)231        #expect(latest.duration >= 0)232233        // A failed run is recorded with its error.234        _ = await runner.run(Workflow(name: "Bad", steps: [.importDocument(path: nil)]), input: .file(folder.file("missing.docx")))235        let afterFailure = store.load()236        #expect(afterFailure.first?.status == .failed)237        #expect(afterFailure.first?.error?.kind == .importFailed)238        #expect(afterFailure.count == 3)239240        store.clear()241        #expect(store.load().isEmpty)242        // A damaged history file does not break anything.243        try Data("{ not json".utf8).write(to: url)244        #expect(store.load().isEmpty)245        store.append(latest)246        #expect(store.load().count == 1)247    }248249    // MARK: JSON validation250251    @Test func jsonWorkflowLoads() throws {252        let json = """253        {254          "name": "Thesis to PDF",255          "summary": "House style for theses.",256          "steps": [257            { "type": "import" },258            { "type": "applyTemplateStyles", "template": "thesis" },259            { "type": "setPageSetup", "paper": "a4", "orientation": "portrait", "margins": 72, "marginLeft": 90 },260            { "type": "acceptAllChanges" },261            { "type": "removeComments" },262            { "type": "tableOfContents", "maxLevel": 2, "title": "Contents" },263            { "type": "setCitationStyle", "style": "apa" },264            { "type": "insertBibliography" },265            { "type": "recalculateTables" },266            { "type": "setMetadata", "author": "A. Student", "keywords": ["thesis"] },267            { "type": "audit", "profile": "academic", "failOnErrors": true },268            { "type": "export", "format": "pdf", "fileName": "pdf/{name}.pdf", "pdf": { "outline": true, "tagged": true, "firstPage": 1, "lastPage": 40 } },269            { "type": "writeAuditReport", "format": "json" }270          ]271        }272        """273        let workflow = try Workflow.load(json: Data(json.utf8))274        #expect(workflow.id == "thesis-to-pdf")275        #expect(workflow.steps.count == 13)276        #expect(workflow.steps[2] == .setPageSetup(PageSetupOptions(paper: "a4", orientation: "portrait", margins: 72, marginLeft: 90)))277        #expect(workflow.steps[10] == .audit(profile: .academic, failOnErrors: true))278        #expect(workflow.steps[11] == .export(format: .pdf, fileName: "pdf/{name}.pdf", pdf: PDFExportOptions(outline: true, tagged: true, firstPage: 1, lastPage: 40)))279        #expect(Set(workflow.steps.map(\.type)).isSubset(of: Set(WorkflowStep.allTypes)))280        #expect(try Workflow.load(json: workflow.jsonData()) == workflow)281    }282283    @Test(arguments: ["runScript", "shell", "exec", "appleScript", "download"])284    func unknownStepTypesAreRejected(type: String) {285        let json = #"{ "name": "Bad", "steps": [ { "type": "import" }, { "type": "\#(type)", "command": "rm -rf ~" } ] }"#286        #expect(throws: WorkflowValidationError.unknownStep(type)) { try Workflow.load(json: Data(json.utf8)) }287    }288289    @Test(arguments: ["../outside.pdf", "/tmp/absolute.pdf", "~/home.pdf", "a/../../b.pdf", "sub/./x.pdf", "a//b.pdf", "..", ""])290    func pathsLeavingTheOutputFolderAreRejected(name: String) throws {291        let data = try JSONSerialization.data(withJSONObject: ["name": "Escape", "steps": [["type": "import"], ["type": "export", "format": "pdf", "fileName": name]]])292        #expect(throws: WorkflowValidationError.pathEscapesOutputFolder(step: 1, path: name)) { try Workflow.load(json: data) }293        let report = try JSONSerialization.data(withJSONObject: ["name": "Escape", "steps": [["type": "writeAuditReport", "format": "md", "fileName": name]]])294        #expect(throws: WorkflowValidationError.pathEscapesOutputFolder(step: 0, path: name)) { try Workflow.load(json: report) }295    }296297    @Test func invalidValuesAreRejectedBeforeAnythingRuns() {298        func load(_ steps: String) throws -> Workflow { try Workflow.load(json: Data(#"{ "name": "X", "steps": [\#(steps)] }"#.utf8)) }299        #expect(throws: WorkflowValidationError.empty) { try load("") }300        #expect(throws: WorkflowValidationError.self) { try load(#"{ "type": "applyTheme", "theme": "Nope" }"#) }301        #expect(throws: WorkflowValidationError.self) { try load(#"{ "type": "applyTheme" }"#) }                       // missing value302        #expect(throws: WorkflowValidationError.self) { try load(#"{ "type": "applyTemplateStyles", "template": "x" }"#) }303        #expect(throws: WorkflowValidationError.self) { try load(#"{ "type": "setPageSetup", "paper": "A9" }"#) }304        #expect(throws: WorkflowValidationError.self) { try load(#"{ "type": "setPageSetup", "margins": -5 }"#) }305        #expect(throws: WorkflowValidationError.self) { try load(#"{ "type": "export", "format": "exe" }"#) }306        #expect(throws: WorkflowValidationError.self) { try load(#"{ "type": "audit", "profile": "strictest" }"#) }307        #expect(throws: WorkflowValidationError.self) { try load(#"{ "type": "setCitationStyle", "style": "made-up" }"#) }308        #expect(throws: WorkflowValidationError.self) { try load(#"{ "type": "tableOfContents", "maxLevel": 12 }"#) }309        #expect(throws: WorkflowValidationError.self) { try load(#"{ "step": "import" }"#) }                         // no type310        #expect(throws: WorkflowValidationError.self) { try Workflow.load(json: Data("not json".utf8)) }311        #expect(throws: WorkflowValidationError.self) { try Workflow.load(json: Data(#"{ "steps": [] }"#.utf8)) }       // no name312    }313314    @Test func outputStaysInsideTheFolderEvenThroughASymbolicLink() async throws {315        let folder = TempFolder()316        let out = folder.subfolder("out"), outside = folder.subfolder("outside")317        try FileManager.default.createDirectory(at: out, withIntermediateDirectories: true)318        try FileManager.default.createDirectory(at: outside, withIntermediateDirectories: true)319        try FileManager.default.createSymbolicLink(at: out.appendingPathComponent("link"), withDestinationURL: outside)320321        let workflow = Workflow(name: "Through a link", steps: [.export(format: .txt, fileName: "link/escaped.txt", pdf: nil)])322        let result = await WorkflowRunner(outputDirectory: out).run(workflow, input: .document(Sample.report(), name: "doc"))323        #expect(result.record.status == .failed)324        #expect(result.record.error?.kind == .pathEscapesOutputFolder)325        #expect(try FileManager.default.contentsOfDirectory(atPath: outside.path).isEmpty)326327        // A workflow built in code with a bad name is refused by the runner as well.328        let direct = await WorkflowRunner(outputDirectory: out).run(Workflow(name: "Direct", steps: [.export(format: .txt, fileName: "../up.txt", pdf: nil)]),329                                                                  input: .document(Sample.report(), name: "doc"))330        #expect(direct.record.error?.kind == .invalidWorkflow)331        #expect(!FileManager.default.fileExists(atPath: folder.file("up.txt").path))332333        // A real subfolder is fine, and an input name cannot smuggle a path in.334        let nested = await WorkflowRunner(outputDirectory: out).run(Workflow(name: "Nested", steps: [.export(format: .txt, fileName: "sub/{name}.txt", pdf: nil)]),335                                                                  input: .document(Sample.report(), name: "../../evil"))336        #expect(nested.record.status == .succeeded, "\(nested.record.error?.description ?? "")")337        let written = try #require(nested.record.outputs.first)338        #expect(written.hasPrefix(out.resolvingSymlinksInPath().path + "/sub/"))339    }340341    @Test func savedWorkflowsLibrary() throws {342        let folder = TempFolder()343        let library = WorkflowLibrary(directory: folder.subfolder("Workflows"))344        #expect(library.entries().isEmpty)345        let url = try library.save(Workflow(name: "House style", steps: [.importDocument(path: nil), .applyTheme(name: "Editorial"), .export(format: .pdf, fileName: nil, pdf: nil)]))346        #expect(url.lastPathComponent == "house-style.json")347        try Data(#"{ "name": "Evil", "steps": [ { "type": "shell", "command": "true" } ] }"#.utf8).write(to: folder.subfolder("Workflows").appendingPathComponent("evil.json"))348        let entries = library.entries()349        #expect(entries.count == 2)350        #expect(entries.first { $0.url.lastPathComponent == "house-style.json" }?.workflow?.steps.count == 3)351        let evil = try #require(entries.first { $0.url.lastPathComponent == "evil.json" })352        #expect(evil.workflow == nil)353        #expect(evil.problem?.contains("shell") == true)354        #expect(throws: WorkflowValidationError.self) { try library.save(Workflow(name: "Empty", steps: [])) }355    }356}357
Files main
Menu

Theme