SPB Git

Repository Public License

CodecTests.swift 19.9 KB 415 lines swift

1// ORIX OS Automation2// File: Packages/ORIXWorkflowEngine/Tests/ORIXWorkflowEngineTests/CodecTests.swift3// Description: Unit and property tests of the YAML/JSON codec: subset coverage, error positions, rejected YAML features, canonical emission and round trips.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Testing9import ORIXCore10@testable import ORIXWorkflowEngine1112/// The illustrative workflow of CLAUDE.md §9, verbatim.13let claudeSection9Workflow = """14apiVersion: orix.ai/v115kind: Automation16metadata:17  name: weekly-research-report18spec:19  trigger:20    type: schedule21    cron: "0 9 * * MON"22    timezone: America/Toronto23  permissions:24    filesystem:25      read: ["~/Documents/Research"]26      write: ["~/Documents/Reports"]27    processes: ["python"]28  budget:29    maxDurationSeconds: 180030    maxProviderCostUSD: 2.0031  steps:32    - id: discover33      tool: filesystem.list34      with:35        path: "~/Documents/Research"36    - id: analyze37      agent: research-analyst38      needs: [discover]39    - id: render40      tool: document.renderPDF41      needs: [analyze]42    - id: verify43      verifier: artifact-exists44      needs: [render]45"""4647func parseYAML(_ text: String) throws -> JSONValue {48    var p = YAMLParser(text)49    return try p.parseDocument().json50}5152func syntaxError(_ text: String) -> YAMLSyntaxError? {53    var p = YAMLParser(text)54    do { _ = try p.parseDocument(); return nil } catch let e as YAMLSyntaxError { return e } catch { return nil }55}5657func minimal(_ stepsYAML: String) -> String {58    "apiVersion: orix.ai/v1\nkind: Automation\nmetadata:\n  id: t\n  name: T\nspec:\n  steps:\n" + stepsYAML59}6061@Suite("YAML subset (unit)") struct YAMLSubsetTests {62    @Test func blockMappingsAndSequences() throws {63        let v = try parseYAML("a: 1\nb:\n  c: [x, y]\n  d:\n    - 1\n    - e: f\n      g: h\nlist:\n- same\n- indent\n")64        #expect(v == ["a": 1, "b": ["c": ["x", "y"], "d": [1, ["e": "f", "g": "h"]]], "list": ["same", "indent"]])65    }6667    @Test func scalarTyping() throws {68        let v = try parseYAML("n: null\nt: ~\ne:\nb1: true\nb2: False\ni: -42\nf: 1.5\ng: 1e3\ns1: 007\ns2: yes\ns3: on\ns4: 0x1F\nq: \"12\"\nbig: 99999999999999999999\n")69        #expect(v["n"] == .null && v["t"] == .null && v["e"] == .null)70        #expect(v["b1"] == true && v["b2"] == false)71        #expect(v["i"] == .int(-42) && v["f"] == .number(1.5) && v["g"] == .number(1000))72        #expect(v["s1"] == "007" && v["s2"] == "yes" && v["s3"] == "on" && v["s4"] == "0x1F" && v["q"] == "12")73        #expect(v["big"] == .number(1e20))74    }7576    @Test func quotedScalarsAndEscapes() throws {77        let v = try parseYAML(#"""78        a: "tab\there \"q\" \\ \n \u00e9 \x41 \U0001F680 \ud83d\ude80 \/"79        b: 'it''s # not a comment: really'80        "quoted key": 'v'81        c: plain # comment82        d: "a # b"83        """#)84        #expect(v["a"] == "tab\there \"q\" \\ \n é A 🚀 🚀 /")85        #expect(v["b"] == "it's # not a comment: really")86        #expect(v["quoted key"] == "v" && v["c"] == "plain" && v["d"] == "a # b")87    }8889    @Test func blockScalars() throws {90        let v = try parseYAML("lit: |\n  line 1\n    indented\n\n  line 3\nstrip: |-\n  no newline\nkeep: |+\n  kept\n\nfold: >\n  folded\n  text\n\n  para\nfolds: >-\n  a\n  b\nlist:\n  - |\n    in list\n  - >-\n    x\n    y\n")91        #expect(v["lit"] == "line 1\n  indented\n\nline 3\n")92        #expect(v["strip"] == "no newline")93        #expect(v["keep"] == "kept\n\n")94        #expect(v["fold"] == "folded text\npara\n")95        #expect(v["folds"] == "a b")96        #expect(v["list"] == ["in list\n", "x y"])97    }9899    @Test func blockScalarKeepsCommentLikeAndColonContent() throws {100        let v = try parseYAML("script: |\n  # not a comment\n  key: value\n  - item\nnext: 1\n")101        #expect(v["script"] == "# not a comment\nkey: value\n- item\n")102        #expect(v["next"] == 1)103    }104105    @Test func flowCollectionsIncludingMultiLineAndJSON() throws {106        let v = try parseYAML("a: [1, two, \"3\", [n, m], {k: v}]\nb: {x: 1, y: [true, null], \"z z\": 'q'}\nc: [\n  1,\n  2 # c\n]\nd: {}\ne: []\n")107        #expect(v["a"] == [1, "two", "3", ["n", "m"], ["k": "v"]])108        #expect(v["b"] == ["x": 1, "y": [true, nil], "z z": "q"])109        #expect(v["c"] == [1, 2] && v["d"] == [:] && v["e"] == [])110        let json = try parseYAML("{\n\t\"a\": [1, 2.5, \"x\\n\"],\n\t\"b\": {\"c\": null, \"d\": false}\n}\n")111        #expect(json == ["a": [1, 2.5, "x\n"], "b": ["c": nil, "d": false]])112    }113114    @Test func commentsAreCollected() throws {115        var p = YAMLParser("# top\na: 1 # trailing\n  # indented\nb: [1, # in flow\n  2]\n")116        _ = try p.parseDocument()117        #expect(p.comments.map(\.line) == [1, 2, 3, 4])118        #expect(p.comments[1].column == 6)119    }120121    @Test(arguments: [122        ("a: &x 1\n", 1, 4, "anchors"),123        ("a: *x\n", 1, 4, "anchors"),124        ("a: !!str 1\n", 1, 4, "tags"),125        ("&k a: 1\n", 1, 1, "anchors"),126        ("a: 1\n---\nb: 2\n", 2, 1, "multiple YAML documents"),127        ("%YAML 1.2\n---\na: 1\n", 1, 1, "directives"),128        ("a: 1\n...\n", 2, 1, "document end marker"),129        ("a:\n  <<: {b: 1}\n", 2, 3, "merge keys"),130        ("? a\n: b\n", 1, 1, "complex mapping keys"),131        ("a: [1, *x]\n", 1, 8, "anchors"),132    ])133    func unsupportedYAMLIsRejectedWithPosition(source: String, line: Int, column: Int, fragment: String) {134        let e = syntaxError(source)135        #expect(e != nil, "expected a rejection for \(source.debugDescription)")136        #expect(e?.line == line && e?.column == column, "got \(e?.line ?? 0):\(e?.column ?? 0) \(e?.message ?? "")")137        #expect(e?.message.contains(fragment) == true, "message: \(e?.message ?? "")")138    }139140    @Test(arguments: [141        ("a: 1\n\tb: 2\n", 2, 1, "tab"),142        ("a: \"open\n", 1, 4, "unterminated"),143        ("a: 'open\n", 1, 4, "unterminated"),144        ("a: [1, 2\n", 1, 4, "unterminated flow"),145        ("a: 1\na: 2\n", 2, 1, "duplicate key"),146        ("a: 1\n  b: 2\n", 2, 3, "unexpected indentation"),147        ("a: \"x\" y\n", 1, 8, "unexpected characters"),148        ("a: b: c\n", 1, 5, "': '"),149        ("a: \"\\q\"\n", 1, 5, "unknown escape"),150        ("a: {b 1}\n", 1, 8, "expected ':'"),151        ("- a\nb: 1\n", 2, 1, "unexpected content"),152        ("a: [1 2: 3]\n", 1, 8, "single-pair"),153    ])154    func syntaxErrorsCarryLineAndColumn(source: String, line: Int, column: Int, fragment: String) {155        let e = syntaxError(source)156        #expect(e != nil, "expected an error for \(source.debugDescription)")157        #expect(e?.line == line && e?.column == column, "got \(e?.line ?? 0):\(e?.column ?? 0) \(e?.message ?? "")")158        #expect(e?.message.contains(fragment) == true, "message: \(e?.message ?? "")")159    }160161    @Test func leadingDocumentMarkerAndCRLFAreAccepted() throws {162        #expect(try parseYAML("---\r\na: 1\r\nb:\r\n  - x\r\n") == ["a": 1, "b": ["x"]])163    }164}165166@Suite("Workflow codec (unit)") struct WorkflowCodecTests {167    @Test func parsesAndNormalisesTheIllustrativeWorkflow() throws {168        let doc = try WorkflowCodec.parse(claudeSection9Workflow)169        #expect(doc.metadata.id == "weekly-research-report" && doc.metadata.name == "weekly-research-report")170        #expect(doc.spec.triggers == [.schedule(cron: "0 9 * * MON", timezone: "America/Toronto")])171        #expect(doc.spec.permissions.filesystem.read == ["~/Documents/Research"])172        #expect(doc.spec.permissions.processes == ["python"])173        #expect(doc.spec.budget == WorkflowBudget(maxDurationSeconds: 1800, maxProviderCostUSD: 2))174        #expect(doc.spec.steps.map(\.id) == ["discover", "analyze", "render", "verify"])175        #expect(doc.spec.steps[0].kind == .tool(ToolStep(tool: "filesystem.list", arguments: ["path": "~/Documents/Research"])))176        #expect(doc.spec.steps[1].kind == .agent(AgentStep(agent: .profile("research-analyst"))))177        #expect(doc.spec.steps[3].kind == .verify(VerifyStep(assertions: [.stepSucceeded("render")])))178        let canonical = WorkflowCodec.emit(doc)179        #expect(canonical.contains("triggers:\n    - type: schedule\n      cron: 0 9 * * MON\n"))180        #expect(canonical.contains("    - id: verify\n      needs: [render]\n      verify:\n        - stepSucceeded: render\n"))181        #expect(try WorkflowCodec.parse(canonical) == doc)182        let loss = WorkflowCodec.roundTripLoss(source: claudeSection9Workflow)183        #expect(loss.contains { $0.contains("'spec.trigger'") })184        #expect(loss.contains { $0.contains("verifier: artifact-exists") })185        #expect(loss.contains { $0.contains("metadata.id") })186    }187188    @Test func everyStepKindParses() throws {189        let src = minimal("""190            - id: t191              name: Tool192              tool: filesystem.list193              with: {path: ~/x, recursive: true}194              retry: {maxAttempts: 3, backoffSeconds: 1.5, backoffMultiplier: 2, force: true}195              timeoutSeconds: 12.5196              continueOnError: true197              onError: f198            - id: a199              agent: {instructions: Do it, model: anthropic/claude-sonnet-4-5, tools: [filesystem.*]}200              objective: Summarise201              inputs: {files: "${{ steps.t.output.entries }}"}202              expect:203                - fileNonEmpty: ~/out.md204            - id: c205              if: ${{ len(steps.t.output.entries) > 0 }}206              then: [a]207              else: [f]208            - id: sw209              switch: ${{ params.mode }}210              cases:211                - value: fast212                  steps: [d]213                - value: 2214                  steps: []215              default: [f]216            - id: fe217              foreach: ${{ steps.t.output.entries }}218              as: entry219              concurrency: 3220              collect: inner221              steps:222                - id: inner223                  transform: ${{ upper(entry.name) }}224            - id: p225              parallel:226                - name: left227                  steps:228                    - id: l1229                      delay: 1230                - name: right231                  steps: []232            - id: lo233              while: ${{ vars.n < 3 }}234              maxIterations: 5235              steps:236                - id: inc237                  set: {n: "${{ vars.n + 1 }}"}238            - id: ap239              approval: Proceed?240            - id: d241              delay: ${{ params.wait }}242            - id: tr243              transform: ${{ 1 + 2 }}244              into: three245            - id: v246              verify:247                - fileExists: ~/a248                - fileContains: ~/a249                  text: hello250                - dirNonEmpty: ~/d251                - equals: [1, "${{ vars.three - 2 }}"]252                - truthy: ${{ vars.three }}253                - stepSucceeded: tr254            - id: sub255              subworkflow: other256              inputs: {x: 1}257            - id: n258              notify: Done259              title: ORIX260            - id: f261              fail: Nope262            - id: o263              output: {count: "${{ steps.fe.output.count }}"}264              artifacts: [~/out.md]265266        """)267        let doc = try WorkflowCodec.parse(src)268        #expect(doc.spec.steps.map(\.kind.name) == ["tool", "agent", "condition", "switch", "foreach", "parallel", "loop", "approval",269                                                    "delay", "transform", "verify", "subworkflow", "notify", "fail", "output"])270        #expect(doc.spec.steps[0].retry == WorkflowRetryPolicy(maxAttempts: 3, backoffSeconds: 1.5, backoffMultiplier: 2, force: true))271        #expect(doc.spec.steps[0].timeoutSeconds == 12.5 && doc.spec.steps[0].continueOnError && doc.spec.steps[0].onError == "f")272        guard case .foreach(let fe) = doc.spec.steps[4].kind else { Issue.record("not foreach"); return }273        #expect(fe.itemName == "entry" && fe.concurrency == 3 && fe.collect == "inner" && fe.steps.count == 1)274        guard case .verify(let v) = doc.spec.steps[10].kind else { Issue.record("not verify"); return }275        #expect(v.assertions.count == 6 && v.assertions[1] == .fileContains(path: "~/a", text: "hello"))276        let text = WorkflowCodec.emit(doc)277        #expect(try WorkflowCodec.parse(text) == doc)278        #expect(WorkflowCodec.emit(try WorkflowCodec.parse(text)) == text)279        let json = WorkflowCodec.emit(doc, format: .json)280        #expect(try WorkflowCodec.parse(json, format: .json) == doc)281        #expect(try WorkflowDocument(jsonValue: JSONValue.parse(json)) == doc)282    }283284    @Test func structuralErrorsCarryPositionPathAndStep() {285        let src = minimal("""286            - id: one287              tool: filesystem.list288              wiht: {path: /x}289            - id: two290              tool: a.b291              agent: x292            - name: no id293              delay: 1294            - id: three295              while: "true"296              steps: []297            - id: four298              timeoutSeconds: soon299              delay: 1300301        """)302        do { _ = try WorkflowCodec.parse(src); Issue.record("should not parse") }303        catch let e as WorkflowParseError {304            let byCode = Dictionary(grouping: e.diagnostics, by: \.code)305            let unknown = byCode["unknown-field"]?.first306            #expect(unknown?.line == 10 && unknown?.column == 7 && unknown?.stepID == "one" && unknown?.path == "spec.steps[0].wiht")307            #expect(byCode["invalid-step-kind"]?.first?.stepID == "two")308            #expect(byCode["invalid-step-kind"]?.first?.line == 11)309            #expect(byCode["missing-field"]?.contains { $0.path == "spec.steps[2]" && $0.line == 14 } == true)310            #expect(byCode["unbounded-loop"]?.first?.stepID == "three")311            let mismatch = byCode["type-mismatch"]?.first312            #expect(mismatch?.path == "spec.steps[4].timeoutSeconds" && mismatch?.line == 20 && mismatch?.column == 23)313        } catch { Issue.record("unexpected error \(error)") }314    }315316    @Test func syntaxErrorsSurfaceAsDiagnostics() {317        #expect(throws: WorkflowParseError.self) { try WorkflowCodec.parse("a: &anchor 1\n") }318        do { _ = try WorkflowCodec.parse("metadata:\n  name: [unclosed\n") }319        catch let e as WorkflowParseError {320            #expect(e.diagnostics.first?.code == "syntax" && e.diagnostics.first?.line == 2 && e.diagnostics.first?.column == 9)321        } catch { Issue.record("unexpected error") }322        #expect(throws: WorkflowParseError.self) { try WorkflowCodec.parse("apiVersion: orix.ai/v9\nkind: Automation\nmetadata: {name: x}\nspec: {steps: []}\n") }323    }324325    @Test func canonicalEmissionUsesMinimalQuotingAndLiteralBlocks() throws {326        var doc = WorkflowDocument(metadata: WorkflowMetadata(id: "q", name: "Quoting: test", description: "line one\nline two\n", tags: ["a", "true", "1"]))327        doc.spec.steps = [WorkflowStep(id: "run", kind: .tool(ToolStep(tool: "shell.script", arguments: [328            "script": "set -e\necho \"hi\"  # comment\n", "path": "~/ORIX Samples/x", "count": "12", "flag": true, "ratio": 2.0, "n": 3,329            "expr": "${{ params.a }}", "list": ["a b", "c,d", "${{ x }}"],330        ])))]331        let text = WorkflowCodec.emit(doc)332        #expect(text.contains("name: \"Quoting: test\"\n"))333        #expect(text.contains("  description: |\n    line one\n    line two\n"))334        #expect(text.contains("tags: [a, \"true\", \"1\"]\n"))335        #expect(text.contains("        script: |\n          set -e\n          echo \"hi\"  # comment\n"))336        #expect(text.contains("        path: ~/ORIX Samples/x\n"))337        #expect(text.contains("        count: \"12\"\n") && text.contains("        flag: true\n") && text.contains("        ratio: 2.0\n") && text.contains("        n: 3\n"))338        #expect(text.contains("        expr: ${{ params.a }}\n"))339        #expect(text.contains("        list: [a b, \"c,d\", \"${{ x }}\"]\n"))340        #expect(try WorkflowCodec.parse(text) == doc)341    }342343    @Test func roundTripLossReportsCommentsFormattingAndNothingForCanonicalText() throws {344        let doc = try WorkflowCodec.parse(claudeSection9Workflow)345        let canonical = WorkflowCodec.emit(doc)346        #expect(WorkflowCodec.roundTripLoss(source: canonical).isEmpty)347        let commented = canonical.replacingOccurrences(of: "spec:\n", with: "spec: # the spec\n  # why this exists\n")348        let loss = WorkflowCodec.roundTripLoss(source: commented)349        #expect(loss.count == 1 && loss[0].hasPrefix("2 comments would be removed"))350        let reordered = "kind: Automation\napiVersion: orix.ai/v1\nmetadata: {id: x, name: 'X'}\nspec:\n  steps: []\n"351        #expect(WorkflowCodec.roundTripLoss(source: reordered).contains { $0.contains("canonical form") })352        #expect(WorkflowCodec.roundTripLoss(source: "a: [").first?.contains("does not parse") == true)353        #expect(WorkflowCodec.roundTripLoss(source: "---\n" + canonical).contains { $0.contains("'---'") })354    }355356    @Test func sourcePositionsLocateValidatorPaths() throws {357        let src = minimal("    - id: a\n      tool: nope.tool\n      with:\n        path: /x\n")358        let positions = WorkflowCodec.sourcePositions(source: src)359        #expect(positions["spec.steps[0].tool"] == WorkflowSourcePosition(line: 9, column: 7))360        #expect(positions["spec.steps[0].with.path"] == WorkflowSourcePosition(line: 11, column: 9))361        let located = WorkflowCodec.locate([WorkflowDiagnostic(code: "x", message: "m", path: "spec.steps[0].with.missing.deeper")], in: src)362        #expect(located[0].line == 10 && located[0].column == 7)363    }364365    @Test func codableBridgeRoundTrips() throws {366        let doc = try WorkflowCodec.parse(claudeSection9Workflow)367        let data = try JSONEncoder().encode(doc)368        #expect(try JSONDecoder().decode(WorkflowDocument.self, from: data) == doc)369        #expect(throws: DecodingError.self) { try JSONDecoder().decode(WorkflowDocument.self, from: Data(#"{"kind":"Nope"}"#.utf8)) }370    }371372    @Test func detectsFormat() {373        #expect(WorkflowCodec.detectFormat("  \n{\"a\": 1}") == .json)374        #expect(WorkflowCodec.detectFormat("apiVersion: x") == .yaml)375    }376}377378@Suite("Workflow codec (property)") struct WorkflowCodecPropertyTests {379    @Test func parseOfEmitIsIdentityForRandomDocuments() throws {380        var generator = DocumentGenerator(seed: 0x0A1C)381        for i in 0..<400 {382            let doc = generator.document()383            for format in WorkflowFormat.allCases {384                let text = WorkflowCodec.emit(doc, format: format)385                let back: WorkflowDocument386                do { back = try WorkflowCodec.parse(text, format: format) }387                catch { Issue.record("iteration \(i) \(format): emitted text does not parse: \(error)\n\(text)"); return }388                if back != doc { Issue.record("iteration \(i) \(format): round trip changed the document\n\(text)"); return }389                #expect(WorkflowCodec.emit(back, format: format) == text)390            }391            #expect(WorkflowCodec.roundTripLoss(source: WorkflowCodec.emit(doc)).isEmpty)392        }393    }394395    @Test func scalarStringsSurviveEveryPosition() throws {396        var generator = DocumentGenerator(seed: 7)397        for s in DocumentGenerator.trickyStrings + (0..<300).map({ _ in generator.string() }) {398            var doc = WorkflowDocument(metadata: WorkflowMetadata(id: s, name: s, description: s, tags: [s, s]))399            doc.spec.variables = [s: .string(s), "list": [.string(s), .string(s)], "nested": ["k": ["deep": .string(s)]]]400            doc.spec.steps = [WorkflowStep(id: "a", name: s, when: s, kind: .tool(ToolStep(tool: "x.y", arguments: [s: .string(s), "a": [.string(s)]])))]401            let text = WorkflowCodec.emit(doc)402            let back = try WorkflowCodec.parse(text)403            #expect(back == doc, "string \(s.debugDescription) did not survive:\n\(text)")404        }405    }406407    @Test func emitOfParseIsIdempotentForHandWrittenSources() throws {408        for src in [claudeSection9Workflow, minimal("    - {id: a, delay: 1}\n    - id: b\n      needs: a\n      set:\n        x:   [1,2]\n")] {409            let once = WorkflowCodec.emit(try WorkflowCodec.parse(src))410            let twice = WorkflowCodec.emit(try WorkflowCodec.parse(once))411            #expect(once == twice)412        }413    }414}415
Files main
Menu

Theme