Repository Public
1// ORIX Docs2// File: Packages/ORIXDocsKit/Tests/DocAutomationTests/WorkflowRunnerTests.swift3// Description: Whole workflows end to end, progress, cancellation, batches with a failing input, history persistence and4// the validation of workflows written as JSON (unknown steps, paths leaving the output folder).5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import PDFKit10import Testing11import DocModel12import DocContainer13@testable import DocAutomation1415@Suite struct WorkflowRunnerTests {16 // MARK: End to end1718 @Test func docxToStyledPDFEndToEnd() async throws {19 let folder = TempFolder()20 let source = folder.file("pandoc-toc.docx")21 try FileManager.default.copyItem(at: Sample.corpus.appendingPathComponent("pandoc-toc.docx"), to: source)22 let out = folder.subfolder("out")23 let events = Collector<WorkflowProgress>()24 let result = await WorkflowRunner(outputDirectory: out).run(BuiltinWorkflows.docxToStyledPDF, input: .file(source)) { events.add($0) }2526 #expect(result.record.status == .succeeded, "\(result.record.error?.description ?? "")")27 #expect(result.record.steps.map(\.type) == ["import", "applyTheme", "tableOfContents", "audit", "export"])28 #expect(result.record.steps.allSatisfy { $0.outcome == .succeeded })29 #expect(result.record.steps.allSatisfy { $0.duration >= 0 })30 #expect(result.record.ended >= result.record.started)31 #expect(result.record.audit != nil)32 #expect(result.document?.styles.themeName == Theme.orix.name)33 #expect(result.document?.body.contains { if case .toc = $0 { return true } else { return false } } == true)3435 let pdf = try #require(PDFDocument(url: out.appendingPathComponent("pandoc-toc.pdf")))36 #expect(pdf.pageCount >= 1)37 // The headings of the source document are in the PDF, in the contents and in the text.38 let imported = try DocumentConverter.importDocument(at: source).document39 let firstHeading = try #require(Numbering(imported).outline.first?.text)40 #expect((pdf.string ?? "").contains(firstHeading))4142 // Progress: a start and a finish per step, in order, ending at 1.43 let progress = events.values44 #expect(progress.count == 10)45 #expect(progress.map(\.stepIndex) == [0, 0, 1, 1, 2, 2, 3, 3, 4, 4])46 #expect(progress.last?.fraction == 1)47 #expect(progress.map(\.fraction) == progress.map(\.fraction).sorted())48 #expect(progress.first?.stepName == "Open the input document")49 }5051 @Test func markdownToPDFWithTemplate() async throws {52 let folder = TempFolder()53 let source = folder.file("field-notes.md")54 try Data(Sample.markdown.utf8).write(to: source)55 let result = await WorkflowRunner(outputDirectory: folder.subfolder("out")).run(BuiltinWorkflows.markdownToPDF, input: .file(source))56 #expect(result.record.status == .succeeded, "\(result.record.error?.description ?? "")")57 #expect(result.document?.pageSetup.paper == .a4)58 let pdf = try #require(PDFDocument(url: folder.subfolder("out").appendingPathComponent("field-notes.pdf")))59 let bounds = try #require(pdf.page(at: 0)?.bounds(for: .mediaBox))60 #expect(abs(bounds.width - 595.28) < 1 && abs(bounds.height - 841.89) < 1)61 #expect((pdf.string ?? "").contains("Field Notes"))62 #expect((pdf.string ?? "").contains("North"))63 }6465 @Test func cleanReviewCopy() async throws {66 let folder = TempFolder()67 var doc = DocumentModel(body: [Sample.para("Budget", style: StyleID.heading(1)), Sample.para("The total is high.")])68 doc.settings.trackChanges = true69 var editor = DocumentEditor(doc: doc, selection: Selection(anchor: Position(block: 1, offset: 13), head: Position(block: 1, offset: 17)), author: "Reviewer")70 editor.insertText("low")71 editor.selection = Selection(anchor: Position(block: 1, offset: 4), head: Position(block: 1, offset: 9))72 editor.addComment("Which total?")73 let source = try Sample.write(editor.doc, to: folder.file("draft.docx"))7475 let result = await WorkflowRunner(outputDirectory: folder.subfolder("out")).run(BuiltinWorkflows.cleanReviewCopy, input: .file(source))76 #expect(result.record.status == .succeeded, "\(result.record.error?.description ?? "")")77 let clean = try DocumentConverter.importDocument(at: folder.subfolder("out").appendingPathComponent("draft-clean.docx")).document78 #expect(!clean.hasTrackedChanges)79 #expect(clean.comments.isEmpty)80 let texts = clean.body.compactMap { $0.paragraph?.plainText }81 #expect(texts.contains("The total is low."))82 }8384 @Test func auditOnlyWritesReportsAndNoDocument() async throws {85 let folder = TempFolder()86 let source = try Sample.write(Sample.report(), to: folder.file("report.orixdoc"))87 let out = folder.subfolder("out")88 let result = await WorkflowRunner(outputDirectory: out).run(BuiltinWorkflows.auditOnly, input: .file(source))89 #expect(result.record.status == .succeeded)90 #expect(try FileManager.default.contentsOfDirectory(atPath: out.path).sorted() == ["report-audit.json", "report-audit.md"])91 }9293 @Test func builtinsAreValidAndListed() throws {94 #expect(BuiltinWorkflows.all.map(\.id) == ["docx-to-styled-pdf", "markdown-to-pdf", "clean-review-copy", "audit-only"])95 for workflow in BuiltinWorkflows.all {96 try workflow.validate()97 #expect(BuiltinWorkflows.named(workflow.id) == workflow)98 // JSON round trip.99 #expect(try Workflow.load(json: workflow.jsonData()) == workflow)100 }101 #expect(BuiltinWorkflows.named("Audit only")?.id == "audit-only")102 #expect(BuiltinWorkflows.named("nope") == nil)103 }104105 // MARK: Cancellation106107 @Test func cancellationStopsBetweenSteps() async throws {108 let folder = TempFolder()109 let steps: [WorkflowStep] = [.applyTheme(name: "Modern"), .export(format: .pdf, fileName: "a.pdf", pdf: nil), .export(format: .docx, fileName: "b.docx", pdf: nil)]110 let runner = WorkflowRunner(outputDirectory: folder.url)111 let box = Collector<Task<WorkflowResult, Never>>()112 let task = Task {113 await runner.run(Workflow(name: "Slow", steps: steps), input: .document(Sample.report(), name: "doc")) { p in114 // Cancel as soon as the first step is done.115 if p.stepIndex == 0, p.phase == .finished { box.values.first?.cancel() }116 }117 }118 box.add(task)119 let result = await task.value120 // The task may have run the first step before `box` held it: accept both, but a cancellation must leave no later output.121 if result.record.status == .cancelled {122 #expect(result.record.error?.kind == .cancelled)123 #expect(result.record.steps.last?.outcome == .notRun || result.record.steps.last?.outcome == .cancelled)124 #expect(!FileManager.default.fileExists(atPath: folder.file("b.docx").path))125 }126127 // Deterministic: a task cancelled before it starts runs no step at all.128 let cancelled = Task { () -> WorkflowResult in129 withUnsafeCurrentTask { $0?.cancel() }130 return await runner.run(Workflow(name: "Slow", steps: steps), input: .document(Sample.report(), name: "never"))131 }132 let none = await cancelled.value133 #expect(none.record.status == .cancelled)134 #expect(none.record.steps.map(\.outcome) == [.cancelled, .notRun, .notRun])135 #expect(none.record.outputs.isEmpty)136 #expect(!FileManager.default.fileExists(atPath: folder.file("never.pdf").path))137 }138139 @Test func batchCancellationStopsStartingFiles() async throws {140 let folder = TempFolder()141 let inputs = try (0..<12).map { try Sample.write(Sample.report(title: "Report \($0)"), to: folder.file("r\($0).orixdoc")) }142 let workflow = Workflow(name: "PDF", steps: [.importDocument(path: nil), .export(format: .pdf, fileName: nil, pdf: nil)])143 let runner = WorkflowRunner(outputDirectory: folder.subfolder("out"))144 let box = Collector<Task<BatchResult, Never>>()145 let task = Task {146 await runner.runBatch(workflow, inputs: inputs, maxConcurrent: 1) { p in147 if p.finished != nil, p.completedFiles == 2 { box.values.first?.cancel() }148 }149 }150 box.add(task)151 let result = await task.value152 #expect(result.runs.count == 12)153 #expect(result.runs.map(\.input) == inputs.map(\.path))154 #expect(result.succeeded.count >= 2)155 #expect(result.cancelled.count >= 8)156 #expect(result.succeeded.count + result.cancelled.count + result.failed.count == 12)157 let written = try FileManager.default.contentsOfDirectory(atPath: folder.subfolder("out").path).count158 #expect(written == result.succeeded.count)159 }160161 // MARK: Batch162163 @Test func batchContinuesAfterAFailingInput() async throws {164 let folder = TempFolder()165 var inputs = try (0..<5).map { try Sample.write(Sample.report(title: "Report \($0)"), to: folder.file("report-\($0).docx")) }166 let broken = folder.file("broken.docx")167 try Data("this is not a zip archive".utf8).write(to: broken)168 inputs.insert(broken, at: 2)169 let out = folder.subfolder("out")170 let events = Collector<BatchProgress>()171 let history = WorkflowHistoryStore(url: folder.file("history.json"))172173 let result = await WorkflowRunner(outputDirectory: out, history: history).runBatch(BuiltinWorkflows.docxToStyledPDF, inputs: inputs, maxConcurrent: 3) { events.add($0) }174175 #expect(result.runs.count == 6)176 #expect(result.runs.map(\.input) == inputs.map(\.path)) // input order, whatever the completion order177 #expect(result.succeeded.count == 5)178 #expect(result.failed.count == 1)179 let failure = try #require(result.failed.first)180 #expect(failure.inputName == "broken.docx")181 #expect(failure.error?.kind == .importFailed)182 #expect(failure.error?.stepIndex == 0)183 #expect(failure.error?.description.contains("broken.docx") == true)184 #expect(failure.steps[0].outcome == .failed && failure.steps[1].outcome == .notRun)185 #expect(try FileManager.default.contentsOfDirectory(atPath: out.path).sorted() == (0..<5).map { "report-\($0).pdf" })186 for i in 0..<5 { #expect(PDFDocument(url: out.appendingPathComponent("report-\(i).pdf"))?.string?.contains("Report \(i)") == true) }187188 let finished = events.values.filter { $0.finished != nil }189 #expect(finished.count == 6)190 #expect(finished.map(\.completedFiles).sorted() == [1, 2, 3, 4, 5, 6])191 #expect(events.values.contains { $0.step != nil })192 #expect(history.load().count == 6)193 }194195 @Test func concurrencyIsBounded() async throws {196 let folder = TempFolder()197 let inputs = try (0..<8).map { try Sample.write(Sample.report(), to: folder.file("d\($0).orixdoc")) }198 let workflow = Workflow(name: "PDF", steps: [.importDocument(path: nil), .export(format: .pdf, fileName: nil, pdf: nil)])199 final class Gauge: @unchecked Sendable {200 let lock = NSLock(); var running = Set<Int>(); var peak = 0201 func update(_ p: BatchProgress) {202 lock.lock(); defer { lock.unlock() }203 if p.finished != nil { running.remove(p.fileIndex) } else { running.insert(p.fileIndex) }204 peak = max(peak, running.count)205 }206 }207 let gauge = Gauge()208 let result = await WorkflowRunner(outputDirectory: folder.subfolder("out")).runBatch(workflow, inputs: inputs, maxConcurrent: 2) { gauge.update($0) }209 #expect(result.succeeded.count == 8)210 #expect(gauge.peak >= 1 && gauge.peak <= 2)211 }212213 // MARK: History214215 @Test func historyPersistsNewestFirstAndIsBounded() async throws {216 let folder = TempFolder()217 let url = folder.subfolder("support").appendingPathComponent("history.json")218 let store = WorkflowHistoryStore(url: url, limit: 3)219 #expect(store.load().isEmpty)220 let runner = WorkflowRunner(outputDirectory: folder.subfolder("out"), existingFiles: .overwrite, history: store)221 for i in 0..<5 {222 _ = await runner.run(Workflow(name: "Run \(i)", steps: [.export(format: .txt, fileName: nil, pdf: nil)]), input: .document(Sample.report(), name: "doc"))223 }224 // A new store on the same file sees what the first one wrote.225 let reopened = WorkflowHistoryStore(url: url, limit: 3).load()226 #expect(reopened.map(\.workflowName) == ["Run 4", "Run 3", "Run 2"])227 let latest = try #require(reopened.first)228 #expect(latest.status == .succeeded)229 #expect(latest.steps.count == 1 && latest.steps[0].outcome == .succeeded)230 #expect(latest.outputs.count == 1)231 #expect(latest.duration >= 0)232233 // A failed run is recorded with its error.234 _ = await runner.run(Workflow(name: "Bad", steps: [.importDocument(path: nil)]), input: .file(folder.file("missing.docx")))235 let afterFailure = store.load()236 #expect(afterFailure.first?.status == .failed)237 #expect(afterFailure.first?.error?.kind == .importFailed)238 #expect(afterFailure.count == 3)239240 store.clear()241 #expect(store.load().isEmpty)242 // A damaged history file does not break anything.243 try Data("{ not json".utf8).write(to: url)244 #expect(store.load().isEmpty)245 store.append(latest)246 #expect(store.load().count == 1)247 }248249 // MARK: JSON validation250251 @Test func jsonWorkflowLoads() throws {252 let json = """253 {254 "name": "Thesis to PDF",255 "summary": "House style for theses.",256 "steps": [257 { "type": "import" },258 { "type": "applyTemplateStyles", "template": "thesis" },259 { "type": "setPageSetup", "paper": "a4", "orientation": "portrait", "margins": 72, "marginLeft": 90 },260 { "type": "acceptAllChanges" },261 { "type": "removeComments" },262 { "type": "tableOfContents", "maxLevel": 2, "title": "Contents" },263 { "type": "setCitationStyle", "style": "apa" },264 { "type": "insertBibliography" },265 { "type": "recalculateTables" },266 { "type": "setMetadata", "author": "A. Student", "keywords": ["thesis"] },267 { "type": "audit", "profile": "academic", "failOnErrors": true },268 { "type": "export", "format": "pdf", "fileName": "pdf/{name}.pdf", "pdf": { "outline": true, "tagged": true, "firstPage": 1, "lastPage": 40 } },269 { "type": "writeAuditReport", "format": "json" }270 ]271 }272 """273 let workflow = try Workflow.load(json: Data(json.utf8))274 #expect(workflow.id == "thesis-to-pdf")275 #expect(workflow.steps.count == 13)276 #expect(workflow.steps[2] == .setPageSetup(PageSetupOptions(paper: "a4", orientation: "portrait", margins: 72, marginLeft: 90)))277 #expect(workflow.steps[10] == .audit(profile: .academic, failOnErrors: true))278 #expect(workflow.steps[11] == .export(format: .pdf, fileName: "pdf/{name}.pdf", pdf: PDFExportOptions(outline: true, tagged: true, firstPage: 1, lastPage: 40)))279 #expect(Set(workflow.steps.map(\.type)).isSubset(of: Set(WorkflowStep.allTypes)))280 #expect(try Workflow.load(json: workflow.jsonData()) == workflow)281 }282283 @Test(arguments: ["runScript", "shell", "exec", "appleScript", "download"])284 func unknownStepTypesAreRejected(type: String) {285 let json = #"{ "name": "Bad", "steps": [ { "type": "import" }, { "type": "\#(type)", "command": "rm -rf ~" } ] }"#286 #expect(throws: WorkflowValidationError.unknownStep(type)) { try Workflow.load(json: Data(json.utf8)) }287 }288289 @Test(arguments: ["../outside.pdf", "/tmp/absolute.pdf", "~/home.pdf", "a/../../b.pdf", "sub/./x.pdf", "a//b.pdf", "..", ""])290 func pathsLeavingTheOutputFolderAreRejected(name: String) throws {291 let data = try JSONSerialization.data(withJSONObject: ["name": "Escape", "steps": [["type": "import"], ["type": "export", "format": "pdf", "fileName": name]]])292 #expect(throws: WorkflowValidationError.pathEscapesOutputFolder(step: 1, path: name)) { try Workflow.load(json: data) }293 let report = try JSONSerialization.data(withJSONObject: ["name": "Escape", "steps": [["type": "writeAuditReport", "format": "md", "fileName": name]]])294 #expect(throws: WorkflowValidationError.pathEscapesOutputFolder(step: 0, path: name)) { try Workflow.load(json: report) }295 }296297 @Test func invalidValuesAreRejectedBeforeAnythingRuns() {298 func load(_ steps: String) throws -> Workflow { try Workflow.load(json: Data(#"{ "name": "X", "steps": [\#(steps)] }"#.utf8)) }299 #expect(throws: WorkflowValidationError.empty) { try load("") }300 #expect(throws: WorkflowValidationError.self) { try load(#"{ "type": "applyTheme", "theme": "Nope" }"#) }301 #expect(throws: WorkflowValidationError.self) { try load(#"{ "type": "applyTheme" }"#) } // missing value302 #expect(throws: WorkflowValidationError.self) { try load(#"{ "type": "applyTemplateStyles", "template": "x" }"#) }303 #expect(throws: WorkflowValidationError.self) { try load(#"{ "type": "setPageSetup", "paper": "A9" }"#) }304 #expect(throws: WorkflowValidationError.self) { try load(#"{ "type": "setPageSetup", "margins": -5 }"#) }305 #expect(throws: WorkflowValidationError.self) { try load(#"{ "type": "export", "format": "exe" }"#) }306 #expect(throws: WorkflowValidationError.self) { try load(#"{ "type": "audit", "profile": "strictest" }"#) }307 #expect(throws: WorkflowValidationError.self) { try load(#"{ "type": "setCitationStyle", "style": "made-up" }"#) }308 #expect(throws: WorkflowValidationError.self) { try load(#"{ "type": "tableOfContents", "maxLevel": 12 }"#) }309 #expect(throws: WorkflowValidationError.self) { try load(#"{ "step": "import" }"#) } // no type310 #expect(throws: WorkflowValidationError.self) { try Workflow.load(json: Data("not json".utf8)) }311 #expect(throws: WorkflowValidationError.self) { try Workflow.load(json: Data(#"{ "steps": [] }"#.utf8)) } // no name312 }313314 @Test func outputStaysInsideTheFolderEvenThroughASymbolicLink() async throws {315 let folder = TempFolder()316 let out = folder.subfolder("out"), outside = folder.subfolder("outside")317 try FileManager.default.createDirectory(at: out, withIntermediateDirectories: true)318 try FileManager.default.createDirectory(at: outside, withIntermediateDirectories: true)319 try FileManager.default.createSymbolicLink(at: out.appendingPathComponent("link"), withDestinationURL: outside)320321 let workflow = Workflow(name: "Through a link", steps: [.export(format: .txt, fileName: "link/escaped.txt", pdf: nil)])322 let result = await WorkflowRunner(outputDirectory: out).run(workflow, input: .document(Sample.report(), name: "doc"))323 #expect(result.record.status == .failed)324 #expect(result.record.error?.kind == .pathEscapesOutputFolder)325 #expect(try FileManager.default.contentsOfDirectory(atPath: outside.path).isEmpty)326327 // A workflow built in code with a bad name is refused by the runner as well.328 let direct = await WorkflowRunner(outputDirectory: out).run(Workflow(name: "Direct", steps: [.export(format: .txt, fileName: "../up.txt", pdf: nil)]),329 input: .document(Sample.report(), name: "doc"))330 #expect(direct.record.error?.kind == .invalidWorkflow)331 #expect(!FileManager.default.fileExists(atPath: folder.file("up.txt").path))332333 // A real subfolder is fine, and an input name cannot smuggle a path in.334 let nested = await WorkflowRunner(outputDirectory: out).run(Workflow(name: "Nested", steps: [.export(format: .txt, fileName: "sub/{name}.txt", pdf: nil)]),335 input: .document(Sample.report(), name: "../../evil"))336 #expect(nested.record.status == .succeeded, "\(nested.record.error?.description ?? "")")337 let written = try #require(nested.record.outputs.first)338 #expect(written.hasPrefix(out.resolvingSymlinksInPath().path + "/sub/"))339 }340341 @Test func savedWorkflowsLibrary() throws {342 let folder = TempFolder()343 let library = WorkflowLibrary(directory: folder.subfolder("Workflows"))344 #expect(library.entries().isEmpty)345 let url = try library.save(Workflow(name: "House style", steps: [.importDocument(path: nil), .applyTheme(name: "Editorial"), .export(format: .pdf, fileName: nil, pdf: nil)]))346 #expect(url.lastPathComponent == "house-style.json")347 try Data(#"{ "name": "Evil", "steps": [ { "type": "shell", "command": "true" } ] }"#.utf8).write(to: folder.subfolder("Workflows").appendingPathComponent("evil.json"))348 let entries = library.entries()349 #expect(entries.count == 2)350 #expect(entries.first { $0.url.lastPathComponent == "house-style.json" }?.workflow?.steps.count == 3)351 let evil = try #require(entries.first { $0.url.lastPathComponent == "evil.json" })352 #expect(evil.workflow == nil)353 #expect(evil.problem?.contains("shell") == true)354 #expect(throws: WorkflowValidationError.self) { try library.save(Workflow(name: "Empty", steps: [])) }355 }356}357