Repository Public License
1// ORIX OS Automation2// File: Packages/ORIXWorkflowEngine/Tests/ORIXWorkflowEngineTests/CodecTests.swift3// Description: Unit and property tests of the YAML/JSON codec: subset coverage, error positions, rejected YAML features, canonical emission and round trips.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Testing9import ORIXCore10@testable import ORIXWorkflowEngine1112/// The illustrative workflow of CLAUDE.md §9, verbatim.13let claudeSection9Workflow = """14apiVersion: orix.ai/v115kind: Automation16metadata:17 name: weekly-research-report18spec:19 trigger:20 type: schedule21 cron: "0 9 * * MON"22 timezone: America/Toronto23 permissions:24 filesystem:25 read: ["~/Documents/Research"]26 write: ["~/Documents/Reports"]27 processes: ["python"]28 budget:29 maxDurationSeconds: 180030 maxProviderCostUSD: 2.0031 steps:32 - id: discover33 tool: filesystem.list34 with:35 path: "~/Documents/Research"36 - id: analyze37 agent: research-analyst38 needs: [discover]39 - id: render40 tool: document.renderPDF41 needs: [analyze]42 - id: verify43 verifier: artifact-exists44 needs: [render]45"""4647func parseYAML(_ text: String) throws -> JSONValue {48 var p = YAMLParser(text)49 return try p.parseDocument().json50}5152func syntaxError(_ text: String) -> YAMLSyntaxError? {53 var p = YAMLParser(text)54 do { _ = try p.parseDocument(); return nil } catch let e as YAMLSyntaxError { return e } catch { return nil }55}5657func minimal(_ stepsYAML: String) -> String {58 "apiVersion: orix.ai/v1\nkind: Automation\nmetadata:\n id: t\n name: T\nspec:\n steps:\n" + stepsYAML59}6061@Suite("YAML subset (unit)") struct YAMLSubsetTests {62 @Test func blockMappingsAndSequences() throws {63 let v = try parseYAML("a: 1\nb:\n c: [x, y]\n d:\n - 1\n - e: f\n g: h\nlist:\n- same\n- indent\n")64 #expect(v == ["a": 1, "b": ["c": ["x", "y"], "d": [1, ["e": "f", "g": "h"]]], "list": ["same", "indent"]])65 }6667 @Test func scalarTyping() throws {68 let v = try parseYAML("n: null\nt: ~\ne:\nb1: true\nb2: False\ni: -42\nf: 1.5\ng: 1e3\ns1: 007\ns2: yes\ns3: on\ns4: 0x1F\nq: \"12\"\nbig: 99999999999999999999\n")69 #expect(v["n"] == .null && v["t"] == .null && v["e"] == .null)70 #expect(v["b1"] == true && v["b2"] == false)71 #expect(v["i"] == .int(-42) && v["f"] == .number(1.5) && v["g"] == .number(1000))72 #expect(v["s1"] == "007" && v["s2"] == "yes" && v["s3"] == "on" && v["s4"] == "0x1F" && v["q"] == "12")73 #expect(v["big"] == .number(1e20))74 }7576 @Test func quotedScalarsAndEscapes() throws {77 let v = try parseYAML(#"""78 a: "tab\there \"q\" \\ \n \u00e9 \x41 \U0001F680 \ud83d\ude80 \/"79 b: 'it''s # not a comment: really'80 "quoted key": 'v'81 c: plain # comment82 d: "a # b"83 """#)84 #expect(v["a"] == "tab\there \"q\" \\ \n é A 🚀 🚀 /")85 #expect(v["b"] == "it's # not a comment: really")86 #expect(v["quoted key"] == "v" && v["c"] == "plain" && v["d"] == "a # b")87 }8889 @Test func blockScalars() throws {90 let v = try parseYAML("lit: |\n line 1\n indented\n\n line 3\nstrip: |-\n no newline\nkeep: |+\n kept\n\nfold: >\n folded\n text\n\n para\nfolds: >-\n a\n b\nlist:\n - |\n in list\n - >-\n x\n y\n")91 #expect(v["lit"] == "line 1\n indented\n\nline 3\n")92 #expect(v["strip"] == "no newline")93 #expect(v["keep"] == "kept\n\n")94 #expect(v["fold"] == "folded text\npara\n")95 #expect(v["folds"] == "a b")96 #expect(v["list"] == ["in list\n", "x y"])97 }9899 @Test func blockScalarKeepsCommentLikeAndColonContent() throws {100 let v = try parseYAML("script: |\n # not a comment\n key: value\n - item\nnext: 1\n")101 #expect(v["script"] == "# not a comment\nkey: value\n- item\n")102 #expect(v["next"] == 1)103 }104105 @Test func flowCollectionsIncludingMultiLineAndJSON() throws {106 let v = try parseYAML("a: [1, two, \"3\", [n, m], {k: v}]\nb: {x: 1, y: [true, null], \"z z\": 'q'}\nc: [\n 1,\n 2 # c\n]\nd: {}\ne: []\n")107 #expect(v["a"] == [1, "two", "3", ["n", "m"], ["k": "v"]])108 #expect(v["b"] == ["x": 1, "y": [true, nil], "z z": "q"])109 #expect(v["c"] == [1, 2] && v["d"] == [:] && v["e"] == [])110 let json = try parseYAML("{\n\t\"a\": [1, 2.5, \"x\\n\"],\n\t\"b\": {\"c\": null, \"d\": false}\n}\n")111 #expect(json == ["a": [1, 2.5, "x\n"], "b": ["c": nil, "d": false]])112 }113114 @Test func commentsAreCollected() throws {115 var p = YAMLParser("# top\na: 1 # trailing\n # indented\nb: [1, # in flow\n 2]\n")116 _ = try p.parseDocument()117 #expect(p.comments.map(\.line) == [1, 2, 3, 4])118 #expect(p.comments[1].column == 6)119 }120121 @Test(arguments: [122 ("a: &x 1\n", 1, 4, "anchors"),123 ("a: *x\n", 1, 4, "anchors"),124 ("a: !!str 1\n", 1, 4, "tags"),125 ("&k a: 1\n", 1, 1, "anchors"),126 ("a: 1\n---\nb: 2\n", 2, 1, "multiple YAML documents"),127 ("%YAML 1.2\n---\na: 1\n", 1, 1, "directives"),128 ("a: 1\n...\n", 2, 1, "document end marker"),129 ("a:\n <<: {b: 1}\n", 2, 3, "merge keys"),130 ("? a\n: b\n", 1, 1, "complex mapping keys"),131 ("a: [1, *x]\n", 1, 8, "anchors"),132 ])133 func unsupportedYAMLIsRejectedWithPosition(source: String, line: Int, column: Int, fragment: String) {134 let e = syntaxError(source)135 #expect(e != nil, "expected a rejection for \(source.debugDescription)")136 #expect(e?.line == line && e?.column == column, "got \(e?.line ?? 0):\(e?.column ?? 0) \(e?.message ?? "")")137 #expect(e?.message.contains(fragment) == true, "message: \(e?.message ?? "")")138 }139140 @Test(arguments: [141 ("a: 1\n\tb: 2\n", 2, 1, "tab"),142 ("a: \"open\n", 1, 4, "unterminated"),143 ("a: 'open\n", 1, 4, "unterminated"),144 ("a: [1, 2\n", 1, 4, "unterminated flow"),145 ("a: 1\na: 2\n", 2, 1, "duplicate key"),146 ("a: 1\n b: 2\n", 2, 3, "unexpected indentation"),147 ("a: \"x\" y\n", 1, 8, "unexpected characters"),148 ("a: b: c\n", 1, 5, "': '"),149 ("a: \"\\q\"\n", 1, 5, "unknown escape"),150 ("a: {b 1}\n", 1, 8, "expected ':'"),151 ("- a\nb: 1\n", 2, 1, "unexpected content"),152 ("a: [1 2: 3]\n", 1, 8, "single-pair"),153 ])154 func syntaxErrorsCarryLineAndColumn(source: String, line: Int, column: Int, fragment: String) {155 let e = syntaxError(source)156 #expect(e != nil, "expected an error for \(source.debugDescription)")157 #expect(e?.line == line && e?.column == column, "got \(e?.line ?? 0):\(e?.column ?? 0) \(e?.message ?? "")")158 #expect(e?.message.contains(fragment) == true, "message: \(e?.message ?? "")")159 }160161 @Test func leadingDocumentMarkerAndCRLFAreAccepted() throws {162 #expect(try parseYAML("---\r\na: 1\r\nb:\r\n - x\r\n") == ["a": 1, "b": ["x"]])163 }164}165166@Suite("Workflow codec (unit)") struct WorkflowCodecTests {167 @Test func parsesAndNormalisesTheIllustrativeWorkflow() throws {168 let doc = try WorkflowCodec.parse(claudeSection9Workflow)169 #expect(doc.metadata.id == "weekly-research-report" && doc.metadata.name == "weekly-research-report")170 #expect(doc.spec.triggers == [.schedule(cron: "0 9 * * MON", timezone: "America/Toronto")])171 #expect(doc.spec.permissions.filesystem.read == ["~/Documents/Research"])172 #expect(doc.spec.permissions.processes == ["python"])173 #expect(doc.spec.budget == WorkflowBudget(maxDurationSeconds: 1800, maxProviderCostUSD: 2))174 #expect(doc.spec.steps.map(\.id) == ["discover", "analyze", "render", "verify"])175 #expect(doc.spec.steps[0].kind == .tool(ToolStep(tool: "filesystem.list", arguments: ["path": "~/Documents/Research"])))176 #expect(doc.spec.steps[1].kind == .agent(AgentStep(agent: .profile("research-analyst"))))177 #expect(doc.spec.steps[3].kind == .verify(VerifyStep(assertions: [.stepSucceeded("render")])))178 let canonical = WorkflowCodec.emit(doc)179 #expect(canonical.contains("triggers:\n - type: schedule\n cron: 0 9 * * MON\n"))180 #expect(canonical.contains(" - id: verify\n needs: [render]\n verify:\n - stepSucceeded: render\n"))181 #expect(try WorkflowCodec.parse(canonical) == doc)182 let loss = WorkflowCodec.roundTripLoss(source: claudeSection9Workflow)183 #expect(loss.contains { $0.contains("'spec.trigger'") })184 #expect(loss.contains { $0.contains("verifier: artifact-exists") })185 #expect(loss.contains { $0.contains("metadata.id") })186 }187188 @Test func everyStepKindParses() throws {189 let src = minimal("""190 - id: t191 name: Tool192 tool: filesystem.list193 with: {path: ~/x, recursive: true}194 retry: {maxAttempts: 3, backoffSeconds: 1.5, backoffMultiplier: 2, force: true}195 timeoutSeconds: 12.5196 continueOnError: true197 onError: f198 - id: a199 agent: {instructions: Do it, model: anthropic/claude-sonnet-4-5, tools: [filesystem.*]}200 objective: Summarise201 inputs: {files: "${{ steps.t.output.entries }}"}202 expect:203 - fileNonEmpty: ~/out.md204 - id: c205 if: ${{ len(steps.t.output.entries) > 0 }}206 then: [a]207 else: [f]208 - id: sw209 switch: ${{ params.mode }}210 cases:211 - value: fast212 steps: [d]213 - value: 2214 steps: []215 default: [f]216 - id: fe217 foreach: ${{ steps.t.output.entries }}218 as: entry219 concurrency: 3220 collect: inner221 steps:222 - id: inner223 transform: ${{ upper(entry.name) }}224 - id: p225 parallel:226 - name: left227 steps:228 - id: l1229 delay: 1230 - name: right231 steps: []232 - id: lo233 while: ${{ vars.n < 3 }}234 maxIterations: 5235 steps:236 - id: inc237 set: {n: "${{ vars.n + 1 }}"}238 - id: ap239 approval: Proceed?240 - id: d241 delay: ${{ params.wait }}242 - id: tr243 transform: ${{ 1 + 2 }}244 into: three245 - id: v246 verify:247 - fileExists: ~/a248 - fileContains: ~/a249 text: hello250 - dirNonEmpty: ~/d251 - equals: [1, "${{ vars.three - 2 }}"]252 - truthy: ${{ vars.three }}253 - stepSucceeded: tr254 - id: sub255 subworkflow: other256 inputs: {x: 1}257 - id: n258 notify: Done259 title: ORIX260 - id: f261 fail: Nope262 - id: o263 output: {count: "${{ steps.fe.output.count }}"}264 artifacts: [~/out.md]265266 """)267 let doc = try WorkflowCodec.parse(src)268 #expect(doc.spec.steps.map(\.kind.name) == ["tool", "agent", "condition", "switch", "foreach", "parallel", "loop", "approval",269 "delay", "transform", "verify", "subworkflow", "notify", "fail", "output"])270 #expect(doc.spec.steps[0].retry == WorkflowRetryPolicy(maxAttempts: 3, backoffSeconds: 1.5, backoffMultiplier: 2, force: true))271 #expect(doc.spec.steps[0].timeoutSeconds == 12.5 && doc.spec.steps[0].continueOnError && doc.spec.steps[0].onError == "f")272 guard case .foreach(let fe) = doc.spec.steps[4].kind else { Issue.record("not foreach"); return }273 #expect(fe.itemName == "entry" && fe.concurrency == 3 && fe.collect == "inner" && fe.steps.count == 1)274 guard case .verify(let v) = doc.spec.steps[10].kind else { Issue.record("not verify"); return }275 #expect(v.assertions.count == 6 && v.assertions[1] == .fileContains(path: "~/a", text: "hello"))276 let text = WorkflowCodec.emit(doc)277 #expect(try WorkflowCodec.parse(text) == doc)278 #expect(WorkflowCodec.emit(try WorkflowCodec.parse(text)) == text)279 let json = WorkflowCodec.emit(doc, format: .json)280 #expect(try WorkflowCodec.parse(json, format: .json) == doc)281 #expect(try WorkflowDocument(jsonValue: JSONValue.parse(json)) == doc)282 }283284 @Test func structuralErrorsCarryPositionPathAndStep() {285 let src = minimal("""286 - id: one287 tool: filesystem.list288 wiht: {path: /x}289 - id: two290 tool: a.b291 agent: x292 - name: no id293 delay: 1294 - id: three295 while: "true"296 steps: []297 - id: four298 timeoutSeconds: soon299 delay: 1300301 """)302 do { _ = try WorkflowCodec.parse(src); Issue.record("should not parse") }303 catch let e as WorkflowParseError {304 let byCode = Dictionary(grouping: e.diagnostics, by: \.code)305 let unknown = byCode["unknown-field"]?.first306 #expect(unknown?.line == 10 && unknown?.column == 7 && unknown?.stepID == "one" && unknown?.path == "spec.steps[0].wiht")307 #expect(byCode["invalid-step-kind"]?.first?.stepID == "two")308 #expect(byCode["invalid-step-kind"]?.first?.line == 11)309 #expect(byCode["missing-field"]?.contains { $0.path == "spec.steps[2]" && $0.line == 14 } == true)310 #expect(byCode["unbounded-loop"]?.first?.stepID == "three")311 let mismatch = byCode["type-mismatch"]?.first312 #expect(mismatch?.path == "spec.steps[4].timeoutSeconds" && mismatch?.line == 20 && mismatch?.column == 23)313 } catch { Issue.record("unexpected error \(error)") }314 }315316 @Test func syntaxErrorsSurfaceAsDiagnostics() {317 #expect(throws: WorkflowParseError.self) { try WorkflowCodec.parse("a: &anchor 1\n") }318 do { _ = try WorkflowCodec.parse("metadata:\n name: [unclosed\n") }319 catch let e as WorkflowParseError {320 #expect(e.diagnostics.first?.code == "syntax" && e.diagnostics.first?.line == 2 && e.diagnostics.first?.column == 9)321 } catch { Issue.record("unexpected error") }322 #expect(throws: WorkflowParseError.self) { try WorkflowCodec.parse("apiVersion: orix.ai/v9\nkind: Automation\nmetadata: {name: x}\nspec: {steps: []}\n") }323 }324325 @Test func canonicalEmissionUsesMinimalQuotingAndLiteralBlocks() throws {326 var doc = WorkflowDocument(metadata: WorkflowMetadata(id: "q", name: "Quoting: test", description: "line one\nline two\n", tags: ["a", "true", "1"]))327 doc.spec.steps = [WorkflowStep(id: "run", kind: .tool(ToolStep(tool: "shell.script", arguments: [328 "script": "set -e\necho \"hi\" # comment\n", "path": "~/ORIX Samples/x", "count": "12", "flag": true, "ratio": 2.0, "n": 3,329 "expr": "${{ params.a }}", "list": ["a b", "c,d", "${{ x }}"],330 ])))]331 let text = WorkflowCodec.emit(doc)332 #expect(text.contains("name: \"Quoting: test\"\n"))333 #expect(text.contains(" description: |\n line one\n line two\n"))334 #expect(text.contains("tags: [a, \"true\", \"1\"]\n"))335 #expect(text.contains(" script: |\n set -e\n echo \"hi\" # comment\n"))336 #expect(text.contains(" path: ~/ORIX Samples/x\n"))337 #expect(text.contains(" count: \"12\"\n") && text.contains(" flag: true\n") && text.contains(" ratio: 2.0\n") && text.contains(" n: 3\n"))338 #expect(text.contains(" expr: ${{ params.a }}\n"))339 #expect(text.contains(" list: [a b, \"c,d\", \"${{ x }}\"]\n"))340 #expect(try WorkflowCodec.parse(text) == doc)341 }342343 @Test func roundTripLossReportsCommentsFormattingAndNothingForCanonicalText() throws {344 let doc = try WorkflowCodec.parse(claudeSection9Workflow)345 let canonical = WorkflowCodec.emit(doc)346 #expect(WorkflowCodec.roundTripLoss(source: canonical).isEmpty)347 let commented = canonical.replacingOccurrences(of: "spec:\n", with: "spec: # the spec\n # why this exists\n")348 let loss = WorkflowCodec.roundTripLoss(source: commented)349 #expect(loss.count == 1 && loss[0].hasPrefix("2 comments would be removed"))350 let reordered = "kind: Automation\napiVersion: orix.ai/v1\nmetadata: {id: x, name: 'X'}\nspec:\n steps: []\n"351 #expect(WorkflowCodec.roundTripLoss(source: reordered).contains { $0.contains("canonical form") })352 #expect(WorkflowCodec.roundTripLoss(source: "a: [").first?.contains("does not parse") == true)353 #expect(WorkflowCodec.roundTripLoss(source: "---\n" + canonical).contains { $0.contains("'---'") })354 }355356 @Test func sourcePositionsLocateValidatorPaths() throws {357 let src = minimal(" - id: a\n tool: nope.tool\n with:\n path: /x\n")358 let positions = WorkflowCodec.sourcePositions(source: src)359 #expect(positions["spec.steps[0].tool"] == WorkflowSourcePosition(line: 9, column: 7))360 #expect(positions["spec.steps[0].with.path"] == WorkflowSourcePosition(line: 11, column: 9))361 let located = WorkflowCodec.locate([WorkflowDiagnostic(code: "x", message: "m", path: "spec.steps[0].with.missing.deeper")], in: src)362 #expect(located[0].line == 10 && located[0].column == 7)363 }364365 @Test func codableBridgeRoundTrips() throws {366 let doc = try WorkflowCodec.parse(claudeSection9Workflow)367 let data = try JSONEncoder().encode(doc)368 #expect(try JSONDecoder().decode(WorkflowDocument.self, from: data) == doc)369 #expect(throws: DecodingError.self) { try JSONDecoder().decode(WorkflowDocument.self, from: Data(#"{"kind":"Nope"}"#.utf8)) }370 }371372 @Test func detectsFormat() {373 #expect(WorkflowCodec.detectFormat(" \n{\"a\": 1}") == .json)374 #expect(WorkflowCodec.detectFormat("apiVersion: x") == .yaml)375 }376}377378@Suite("Workflow codec (property)") struct WorkflowCodecPropertyTests {379 @Test func parseOfEmitIsIdentityForRandomDocuments() throws {380 var generator = DocumentGenerator(seed: 0x0A1C)381 for i in 0..<400 {382 let doc = generator.document()383 for format in WorkflowFormat.allCases {384 let text = WorkflowCodec.emit(doc, format: format)385 let back: WorkflowDocument386 do { back = try WorkflowCodec.parse(text, format: format) }387 catch { Issue.record("iteration \(i) \(format): emitted text does not parse: \(error)\n\(text)"); return }388 if back != doc { Issue.record("iteration \(i) \(format): round trip changed the document\n\(text)"); return }389 #expect(WorkflowCodec.emit(back, format: format) == text)390 }391 #expect(WorkflowCodec.roundTripLoss(source: WorkflowCodec.emit(doc)).isEmpty)392 }393 }394395 @Test func scalarStringsSurviveEveryPosition() throws {396 var generator = DocumentGenerator(seed: 7)397 for s in DocumentGenerator.trickyStrings + (0..<300).map({ _ in generator.string() }) {398 var doc = WorkflowDocument(metadata: WorkflowMetadata(id: s, name: s, description: s, tags: [s, s]))399 doc.spec.variables = [s: .string(s), "list": [.string(s), .string(s)], "nested": ["k": ["deep": .string(s)]]]400 doc.spec.steps = [WorkflowStep(id: "a", name: s, when: s, kind: .tool(ToolStep(tool: "x.y", arguments: [s: .string(s), "a": [.string(s)]])))]401 let text = WorkflowCodec.emit(doc)402 let back = try WorkflowCodec.parse(text)403 #expect(back == doc, "string \(s.debugDescription) did not survive:\n\(text)")404 }405 }406407 @Test func emitOfParseIsIdempotentForHandWrittenSources() throws {408 for src in [claudeSection9Workflow, minimal(" - {id: a, delay: 1}\n - id: b\n needs: a\n set:\n x: [1,2]\n")] {409 let once = WorkflowCodec.emit(try WorkflowCodec.parse(src))410 let twice = WorkflowCodec.emit(try WorkflowCodec.parse(once))411 #expect(once == twice)412 }413 }414}415