SPB Git

Repository Public License

ValidatorTests.swift 41.7 KB 988 lines swift

1// ORIX OS Automation2// File: Packages/ORIXWorkflowEngine/Tests/ORIXWorkflowEngineTests/ValidatorTests.swift3// Description: Unit tests of the static validator: one focused case per diagnostic code, plus capabilities, proposed grants and source positions.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Testing9import ORIXCore10@testable import ORIXWorkflowEngine1112/// Builds a document from the body of `spec:` (written at two-space indentation) and validates it.13private func check(_ spec: String, tools: [ToolManifest]? = nil, agents: [String] = [], workflows: [String] = [],14                   sourceLocation: SourceLocation = #_sourceLocation) -> [WorkflowDiagnostic] {15    let source = "apiVersion: orix.ai/v1\nkind: Automation\nmetadata:\n  id: t\n  name: T\nspec:\n" + spec16    do {17        let doc = try WorkflowCodec.parse(source)18        return WorkflowValidator.validate(doc, tools: tools ?? validatorTools, agents: agents, workflows: workflows)19    } catch {20        Issue.record("test source does not parse: \(error.localizedDescription)", sourceLocation: sourceLocation)21        return []22    }23}2425/// The canonical catalogue plus one tool with a strict input schema.26private let validatorTools: [ToolManifest] = CanonicalTools.manifests() + [27    ToolManifest(identifier: "test.strict", name: "Strict", summary: "Tool with a strict input schema for validator tests.", family: "test",28                 inputSchema: .object(["path": .string(), "count": .integer(minimum: 1), "tags": .array(of: .string())], required: ["path"]),29                 sideEffect: .read, defaultTier: .readOnly, idempotent: true),30]3132private extension Array where Element == WorkflowDiagnostic {33    func with(_ code: String) -> [WorkflowDiagnostic] { filter { $0.code == code } }34    func has(_ code: String, _ severity: WorkflowDiagnostic.Severity = .error, step: String? = nil, containing text: String = "") -> Bool {35        contains { $0.code == code && $0.severity == severity && (step == nil || $0.stepID == step) && (text.isEmpty || $0.message.contains(text)) }36    }37    var errors: [WorkflowDiagnostic] { filter { $0.severity == .error } }38    var summary: String { map(\.description).joined(separator: "\n") }39}4041@Suite("Workflow validator: identifiers and graph (unit)") struct ValidatorGraphTests {42    @Test func cleanDocumentHasNoDiagnostics() {43        let d = check("""44          steps:45            - id: a46              delay: 147            - id: b48              needs: [a]49              transform: ${{ steps.a.status }}50        """)51        #expect(d.isEmpty, "\(d.summary)")52    }5354    @Test func emptyWorkflowIsAWarning() {55        #expect(check("  steps: []\n").has("empty-workflow", .warning))56    }5758    @Test func invalidIdentifiers() {59        let d = check("""60          steps:61            - id: 1bad62              delay: 163            - id: "has space"64              delay: 165            - id: ok-id_266              delay: 167        """)68        #expect(d.with("invalid-id").map(\.stepID) == ["1bad", "has space"])69        #expect(d.with("invalid-id").first?.path == "spec.steps[0].id")70        var doc = WorkflowDocument(metadata: WorkflowMetadata(id: "bad id", name: " "))71        doc.spec.steps = [WorkflowStep(id: "a", kind: .delay(DelayStep(seconds: 1)))]72        let m = WorkflowValidator.validate(doc, tools: [], agents: [], workflows: [])73        #expect(m.has("invalid-id", containing: "metadata.id") && m.has("missing-field", containing: "metadata.name"))74        #expect(WorkflowValidator.isValidIdentifier("a-b_1") && !WorkflowValidator.isValidIdentifier("-a") && !WorkflowValidator.isValidIdentifier(""))75    }7677    @Test func duplicateIdsAreDocumentWideIncludingNestedSteps() {78        let d = check("""79          steps:80            - id: a81              delay: 182            - id: loop83              foreach: [1, 2]84              steps:85                - id: a86                  delay: 187        """)88        #expect(d.has("duplicate-id", step: "a", containing: "spec.steps[0]"))89        #expect(d.with("duplicate-id").first?.path == "spec.steps[1].steps[0].id")90    }9192    @Test func selfNeedAndUnknownNeed() {93        let d = check("""94          steps:95            - id: a96              needs: [a, ghost]97              delay: 198            - id: box99              foreach: [1]100              steps:101                - id: inner102                  delay: 1103            - id: b104              needs: [inner]105              delay: 1106        """)107        #expect(d.has("self-need", step: "a"))108        #expect(d.has("unknown-need", step: "a", containing: "'ghost'"))109        #expect(d.has("unknown-need", step: "b", containing: "another scope"))110        #expect(d.with("unknown-need").first?.path == "spec.steps[0].needs[1]")111    }112113    @Test func unknownBranchStep() {114        let d = check("""115          steps:116            - id: c117              if: "true"118              then: [ghost]119              else: [c]120            - id: s121              switch: "1"122              cases:123                - value: 1124                  steps: [nowhere]125        """)126        #expect(d.with("unknown-branch-step").count == 3)127        #expect(d.has("unknown-branch-step", step: "s", containing: "'nowhere'"))128    }129130    @Test func dependencyCycleReportsItsPath() {131        let d = check("""132          steps:133            - id: a134              needs: [b]135              delay: 1136            - id: b137              needs: [c]138              delay: 1139            - id: c140              needs: [a]141              delay: 1142            - id: free143              delay: 1144        """)145        let cycles = d.with("dependency-cycle")146        #expect(cycles.count == 1)147        #expect(cycles.first?.message.contains("c → b → a → c") == true, "\(cycles.first?.message ?? "")")148        #expect(cycles.first?.path == "spec.steps[0].needs" && cycles.first?.stepID == "a")149    }150151    @Test func cycleThroughAConditionGateIsDetected() {152        let d = check("""153          steps:154            - id: gate155              needs: [target]156              if: "true"157              then: [target]158            - id: target159              delay: 1160        """)161        #expect(d.has("dependency-cycle"))162    }163164    @Test func structuralErrorsOnlyReportGraphProblems() throws {165        let doc = try WorkflowCodec.parse("apiVersion: orix.ai/v1\nkind: Automation\nmetadata: {id: t, name: T}\nspec:\n  steps:\n    - id: a\n      needs: [zz]\n      tool: nope.tool\n")166        #expect(WorkflowValidator.structuralErrors(doc).map(\.code) == ["unknown-need"])167    }168169    @Test func unreachableSteps() {170        let d = check("""171          steps:172            - id: work173              delay: 1174              onError: handler175            - id: handler176              delay: 1177            - id: afterHandler178              needs: [handler]179              delay: 1180            - id: stop181              fail: Always182            - id: afterFail183              needs: [stop]184              delay: 1185            - id: later186              needs: [afterFail]187              delay: 1188            - id: gate189              if: "false"190              then: [never]191              else: [always]192            - id: never193              delay: 1194            - id: always195              delay: 1196        """)197        #expect(d.has("unreachable-step", .error, step: "afterHandler", containing: "onError handler"))198        #expect(d.has("unreachable-step", .warning, step: "afterFail", containing: "unconditional fail step 'stop'"))199        #expect(d.has("unreachable-step", .warning, step: "later"))200        #expect(d.has("unreachable-step", .warning, step: "never", containing: "always false"))201        #expect(!d.with("unreachable-step").contains { $0.stepID == "always" || $0.stepID == "handler" || $0.stepID == "work" })202        let handled = check("""203          steps:204            - id: stop205              fail: Sometimes206              continueOnError: true207            - id: after208              needs: [stop]209              delay: 1210        """)211        #expect(handled.with("unreachable-step").isEmpty)212    }213214    @Test func invalidHandlers() {215        let d = check("""216          steps:217            - id: a218              delay: 1219              onError: a220            - id: b221              delay: 1222              onError: ghost223            - id: c224              delay: 1225              onError: needy226            - id: needy227              needs: [a]228              delay: 1229            - id: d230              delay: 1231              onError: gated232            - id: gate233              if: "true"234              then: [gated]235            - id: gated236              delay: 1237              onError: last238            - id: last239              delay: 1240        """)241        let h = d.with("invalid-handler")242        #expect(h.contains { $0.stepID == "a" && $0.message.contains("its own") })243        #expect(h.contains { $0.stepID == "b" && $0.message.contains("'ghost'") })244        #expect(h.contains { $0.stepID == "c" && $0.message.contains("must not declare 'needs'") })245        #expect(h.contains { $0.stepID == "d" && $0.message.contains("branch") })246        #expect(h.contains { $0.stepID == "d" && $0.severity == .warning && $0.message.contains("not chained") })247    }248249    @Test func staleLayoutEntry() {250        let d = check("""251          steps:252            - id: a253              delay: 1254          layout:255            a: {x: 1, y: 2}256            gone: {x: 3, y: 4}257            "trigger:0": {x: 0, y: 0}258        """)259        #expect(d.with("stale-layout").map(\.path) == ["spec.layout.gone"])260    }261}262263@Suite("Workflow validator: tools, arguments and step rules (unit)") struct ValidatorStepTests {264    @Test func unknownToolNamesItsFamily() {265        let d = check("""266          steps:267            - id: a268              tool: filesystem.lst269            - id: b270              tool: document.renderPDF271        """)272        #expect(d.has("unknown-tool", step: "a", containing: "filesystem.list"))273        #expect(d.has("unknown-tool", step: "b"))274        #expect(d.with("unknown-tool").first?.path == "spec.steps[0].tool")275    }276277    @Test func literalArgumentsAreCheckedAgainstTheInputSchema() {278        let d = check("""279          steps:280            - id: missing281              tool: test.strict282              with: {count: 2}283            - id: unexpected284              tool: test.strict285              with: {path: /x, colour: red}286            - id: wrongType287              tool: test.strict288              with: {path: 12, count: 0, tags: [a, 3]}289            - id: dynamic290              tool: test.strict291              with:292                path: ${{ params.p }}293                count: ${{ params.n }}294                tags: ["${{ params.p }}", 4]295          parameters:296            - {name: p, type: string, default: /x}297            - {name: n, type: integer, default: 1}298        """)299        let a = d.with("invalid-argument")300        #expect(a.contains { $0.stepID == "missing" && $0.message.contains("requires argument 'path'") })301        #expect(a.contains { $0.stepID == "unexpected" && $0.message.contains("no argument 'colour'") && $0.path == "spec.steps[1].with.colour" })302        #expect(a.contains { $0.stepID == "wrongType" && $0.message.contains("'path'") && $0.message.contains("expected string") })303        #expect(a.contains { $0.stepID == "wrongType" && $0.message.contains("'count'") && $0.message.contains("minimum") })304        #expect(a.contains { $0.stepID == "wrongType" && $0.message.contains("'tags.1'") })305        #expect(!a.contains { $0.stepID == "dynamic" }, "values containing ${{ }} are only known at run time")306    }307308    @Test func unboundedLoops() {309        let d = check("""310          steps:311            - id: zero312              while: ${{ vars.go }}313              maxIterations: 0314              steps: [{id: z1, delay: 1}]315            - id: huge316              while: ${{ vars.go }}317              maxIterations: 50000318              steps: [{id: h1, delay: 1}]319            - id: constant320              while: "true"321              maxIterations: 3322              steps: [{id: c1, delay: 1}]323          variables: {go: true}324        """)325        #expect(d.has("unbounded-loop", .error, step: "zero"))326        #expect(d.has("unbounded-loop", .warning, step: "huge", containing: "effectively unbounded"))327        #expect(d.has("unbounded-loop", .warning, step: "constant", containing: "constant"))328    }329330    @Test func invalidValuesOnSteps() {331        let d = check("""332          steps:333            - id: t334              delay: -1335              timeoutSeconds: 0336              retry: {maxAttempts: 0, backoffSeconds: -1}337            - id: fe338              foreach: [1]339              as: steps340              concurrency: 0341              collect: ghost342              steps: []343            - id: st344              set: {"bad name": 1}345            - id: tr346              transform: "1"347              into: 9x348            - id: empty349              set: {}350            - id: v351              verify: []352            - id: par353              parallel:354                - name: same355                  steps: []356                - name: same357                  steps: [{id: p1, delay: 1}]358        """)359        #expect(d.with("invalid-value").filter { $0.stepID == "t" }.count == 4)360        #expect(d.has("invalid-value", step: "fe", containing: "concurrency"))361        #expect(d.has("invalid-value", step: "fe", containing: "'as: steps'"))362        #expect(d.has("unknown-step-reference", step: "fe", containing: "'collect'"))363        #expect(d.has("empty-body", .warning, step: "fe") && d.has("empty-body", .warning, step: "empty") && d.has("empty-body", .warning, step: "par"))364        #expect(d.has("invalid-id", step: "st") && d.has("invalid-id", step: "tr"))365        #expect(d.has("empty-verify", .warning, step: "v"))366        #expect(d.has("duplicate-branch", step: "par"))367    }368369    @Test func conditionAndSwitchWarnings() {370        let d = check("""371          steps:372            - id: c373              if: ${{ vars.x }}374            - id: both375              if: ${{ vars.x }}376              then: [t]377              else: [t]378            - id: t379              delay: 1380            - id: s381              switch: ${{ vars.x }}382              cases:383                - {value: 1, steps: [u]}384                - {value: 1, steps: []}385            - id: u386              delay: 1387            - id: s2388              switch: ${{ vars.x }}389          variables: {x: 1}390        """)391        #expect(d.has("empty-branches", .warning, step: "c") && d.has("empty-branches", .warning, step: "s2"))392        #expect(d.has("ambiguous-branch", .warning, step: "both"))393        #expect(d.has("duplicate-case", .warning, step: "s"))394    }395396    @Test func retryOnNonIdempotentStepsIsFlagged() {397        let d = check("""398          steps:399            - id: read400              tool: filesystem.list401              with: {path: /tmp/x}402              retry: 3403            - id: move404              tool: filesystem.move405              with: {source: /tmp/x/a, destination: /tmp/x/b}406              retry: {maxAttempts: 3}407            - id: forced408              tool: filesystem.move409              with: {source: /tmp/x/a, destination: /tmp/x/b}410              retry: {maxAttempts: 3, force: true}411            - id: ag412              agent: {instructions: Do it}413              objective: Something414              retry: 2415            - id: box416              foreach: [1]417              retry: 2418              steps: [{id: b1, delay: 1}]419          permissions:420            filesystem: {read: [/tmp/x], write: [/tmp/x], delete: [/tmp/x]}421          budget: {maxProviderCostUSD: 1}422        """)423        let r = d.with("retry-not-idempotent")424        #expect(r.map(\.stepID) == ["move", "ag", "box"], "\(r.summary)")425        #expect(r.allSatisfy { $0.severity == .warning } && r[0].message.contains("'filesystem.move' is not idempotent"))426    }427428    @Test func agentRules() {429        let d = check("""430          steps:431            - id: named432              agent: research-analyst433              objective: Research434            - id: unknown435              agent: nobody436              objective: Research437            - id: inline438              agent: {instructions: "  ", model: sonnet, tools: [filesystem.*, nope.*, "*"]}439              expect:440                - fileExists: ~/a441                - truthy: "true"442          budget: {maxProviderCostUSD: 1}443        """, agents: ["research-analyst"])444        #expect(d.with("unknown-agent").map(\.stepID) == ["unknown"])445        #expect(d.has("missing-field", step: "inline", containing: "instructions"))446        #expect(d.has("invalid-model", step: "inline", containing: "provider/model"))447        #expect(d.with("unknown-tool").map(\.path) == ["spec.steps[2].agent.tools[1]"])448        #expect(d.has("missing-objective", .warning, step: "inline") && !d.has("missing-objective", .warning, step: "named"))449        #expect(d.with("invalid-assertion").map(\.path) == ["spec.steps[2].expect[1]"])450    }451452    @Test func subworkflowRules() {453        let d = check("""454          steps:455            - id: ok456              subworkflow: shared-cleanup457            - id: missing458              subworkflow: nowhere459            - id: selfByID460              subworkflow: t461            - id: selfByName462              subworkflow: T463        """, workflows: ["shared-cleanup"])464        #expect(d.with("unknown-workflow").map(\.stepID) == ["missing"])465        #expect(d.with("recursive-subworkflow").map(\.stepID) == ["selfByID", "selfByName"])466    }467}468469@Suite("Workflow validator: expressions (unit)") struct ValidatorExpressionTests {470    @Test func syntaxErrorsAndUnknownFunctions() {471        let d = check("""472          steps:473            - id: a474              when: ${{ 1 + }}475              delay: 1476            - id: b477              transform: nope(1) + len("x")478            - id: c479              tool: filesystem.list480              with: {path: "/tmp/${{ unterminated"}481            - id: d482              if: " "483              then: [a]484        """)485        #expect(d.has("expression-syntax", step: "a") && d.with("expression-syntax").first?.path == "spec.steps[0].when")486        #expect(d.has("unknown-function", step: "b", containing: "'nope'") && d.with("unknown-function").count == 1)487        #expect(d.has("expression-syntax", step: "c", containing: "unterminated"))488        #expect(d.has("expression-syntax", step: "d", containing: "empty expression"))489    }490491    @Test func parametersVariablesEnvironmentAndNames() {492        let d = check("""493          parameters:494            - {name: declared, type: string, default: x}495          variables: {known: 1}496          steps:497            - id: a498              set:499                later: ${{ params.declared + params.missing }}500            - id: b501              needs: [a]502              transform: ${{ vars.known + vars.later + vars.never }}503              into: fromTransform504            - id: c505              needs: [b]506              transform: ${{ vars.fromTransform }}507            - id: d508              transform: ${{ env.HOME + env.PATH + env["SHELL"] }}509            - id: e510              transform: ${{ item + stranger + map([1], x => x + other) }}511        """)512        #expect(d.with("missing-parameter").count == 1 && d.has("missing-parameter", step: "a", containing: "'missing'"))513        #expect(d.with("unset-variable").count == 1 && d.has("unset-variable", .warning, step: "b", containing: "vars.never"))514        #expect(d.with("env-not-allowed").map { $0.message.contains("env.PATH") || $0.message.contains("env.SHELL") } == [true, true])515        let names = d.with("unknown-name")516        #expect(names.count == 3 && names.allSatisfy { $0.stepID == "e" })517        #expect(!names.contains { $0.message.contains("'x'") }, "lambda parameters are bound")518    }519520    @Test func stepReferencesMustBeUpstream() {521        let d = check("""522          steps:523            - id: first524              delay: 1525            - id: sibling526              transform: ${{ steps.first.status }}527            - id: ghostRef528              transform: ${{ steps.ghost.output }}529            - id: selfRef530              transform: ${{ steps.selfRef.output }}531            - id: box532              needs: [first]533              foreach: [1]534              steps:535                - id: inner536                  transform: ${{ steps.first.status }}537            - id: outside538              needs: [box]539              transform: ${{ steps.inner.output }}540            - id: v541              verify:542                - stepSucceeded: first543                - stepSucceeded: ghost544        """)545        let up = d.with("step-not-upstream")546        #expect(up.contains { $0.stepID == "sibling" && $0.message.contains("add it to 'needs'") })547        #expect(up.contains { $0.stepID == "selfRef" && $0.message.contains("its own result") })548        #expect(up.contains { $0.stepID == "outside" && $0.message.contains("container's output") })549        #expect(up.contains { $0.stepID == "v" && $0.path == "spec.steps[6].verify[0].stepSucceeded" })550        #expect(!up.contains { $0.stepID == "inner" }, "a nested step sees what its container sees")551        #expect(d.with("unknown-step-reference").map(\.stepID) == ["ghostRef", "v"])552    }553554    @Test func allowedReferencesProduceNoDiagnostic() {555        let d = check("""556          variables: {n: 0}557          steps:558            - id: a559              delay: 1560            - id: b561              needs: [a]562              delay: 1563            - id: transitive564              needs: [b]565              transform: ${{ steps.a.status + steps.b.status }}566            - id: gate567              needs: [a]568              if: ${{ steps.a.status == "succeeded" }}569              then: [gated]570            - id: gated571              transform: ${{ steps.gate.output.result && steps.a.status == "succeeded" }}572            - id: each573              needs: [a]574              foreach: [1, 2]575              as: thing576              collect: use577              steps:578                - id: use579                  transform: ${{ thing + index + len(steps.a.status) }}580            - id: again581              while: ${{ loop.index < 3 && steps.bump.output != 2 && default(steps.peek.output, 0) < 5 }}582              maxIterations: 5583              steps:584                - id: bump585                  transform: ${{ loop.index + default(steps.peek.output, 0) }}586                - id: peek587                  transform: ${{ loop.index }}588            - id: risky589              delay: 1590              onError: recover591            - id: recover592              transform: ${{ error.message + steps.risky.status }}593          onError:594            - id: report595              transform: ${{ error.stepID + steps.a.status + steps.transitive.status }}596        """)597        #expect(d.isEmpty, "\(d.summary)")598    }599600    @Test func loopVariablesAreScoped() {601        let d = check("""602          steps:603            - id: each604              foreach: [1]605              steps:606                - id: inside607                  transform: ${{ item }}608            - id: outside609              needs: [each]610              transform: ${{ item }}611            - id: noError612              transform: ${{ error.message }}613            - id: noLoop614              transform: ${{ loop.index }}615        """)616        #expect(d.with("unknown-name").map(\.stepID) == ["outside", "noError", "noLoop"])617    }618}619620@Suite("Workflow validator: triggers, parameters, permissions, budget (unit)") struct ValidatorSpecTests {621    @Test(arguments: [622        ("0 9 * * MON", true), ("*/15 * * * *", true), ("0 9-17 * * MON-FRI", true), ("5,35 0 1 jan,jul *", true), ("0 0 * * 7", true),623        ("0 9 * *", false), ("60 * * * *", false), ("* 24 * * *", false), ("0 0 0 * *", false), ("0 0 * 13 *", false),624        ("0 0 * * FUNDAY", false), ("*/0 * * * *", false), ("10-5 * * * *", false), ("0 9 * * MON extra", false), ("a b c d e", false),625    ])626    func cronExpressions(cron: String, valid: Bool) {627        #expect((WorkflowValidator.cronProblem(cron) == nil) == valid, "\(cron): \(WorkflowValidator.cronProblem(cron) ?? "ok")")628    }629630    @Test func invalidTriggers() {631        let d = check("""632          triggers:633            - {type: schedule, cron: "61 * * * *", timezone: America/Toronto}634            - {type: schedule, cron: "0 9 * * *", timezone: Mars/Olympus}635            - {type: once, at: tomorrow}636            - {type: once, at: "2026-12-01T09:00:00-05:00"}637            - {type: interval, seconds: 0}638            - {type: interval, seconds: 5}639            - {type: fileChange, path: relative/dir, debounceSeconds: -1}640            - {type: appLaunch, bundleID: Safari}641            - {type: fileChange, path: ~/ok}642          steps:643            - {id: a, delay: 1}644        """)645        #expect(d.with("invalid-cron").map(\.path) == ["spec.triggers[0].cron"])646        #expect(d.with("invalid-timezone").map(\.path) == ["spec.triggers[1].timezone"])647        let t = d.with("invalid-trigger")648        #expect(t.map(\.path) == ["spec.triggers[2].at", "spec.triggers[4].seconds", "spec.triggers[5].seconds", "spec.triggers[6].path",649                                  "spec.triggers[6].debounceSeconds", "spec.triggers[7].bundleID"], "\(t.summary)")650        #expect(t[2].severity == .warning && t[1].severity == .error)651    }652653    @Test func parameterRules() {654        let d = check("""655          parameters:656            - {name: a, type: integer, default: 1}657            - {name: a, type: string}658            - {name: bad-name, type: string}659            - {name: n, type: number, default: nope}660            - {name: i, type: integer, default: 1.5}661            - {name: p, type: path, default: 3}662            - {name: ok, type: array, default: [1]}663            - {name: nul, type: object, default: null}664            - {name: req, type: boolean, default: true, required: true}665          steps:666            - {id: a, delay: 1}667        """)668        #expect(d.with("duplicate-parameter").map(\.path) == ["spec.parameters[1].name"])669        #expect(d.has("invalid-id", containing: "'bad-name'"))670        #expect(d.with("parameter-default-type").map(\.path) == ["spec.parameters[3].default", "spec.parameters[4].default", "spec.parameters[5].default"])671        #expect(d.has("parameter-required-default", .info))672    }673674    @Test func permissionEntriesMustBeAbsolutePathsAndBareHosts() {675        let d = check("""676          parameters:677            - {name: folder, type: path, default: ~/Work}678            - {name: rel, type: path, default: work/here}679          permissions:680            filesystem:681              read: [relative/path, "~bob/Documents", "", "${{ params.folder }}/in", "${{ params.rel }}"]682              write: [/, "~", "${{ params.ghost }}", "${{ vars.x }}/y", "${{ steps.a.output }}", "${{ 1 + }}"]683            network: ["https://example.com", "example.com/path", "example.com:8080", "*.example.com", "*", ""]684            browser: [ok.example.com, "bad host"]685            scripts: [bash, ruby]686            processes: [git, ""]687          steps:688            - {id: a, delay: 1}689        """)690        let paths = d.with("invalid-permission-path")691        #expect(paths.map(\.path) == ["spec.permissions.filesystem.read[0]", "spec.permissions.filesystem.read[1]", "spec.permissions.filesystem.read[2]",692                                      "spec.permissions.filesystem.read[4]"], "\(paths.summary)")693        #expect(paths[0].message.contains("relative") && paths[1].message.contains("~user") && paths[2].message.contains("empty"))694        #expect(d.with("broad-permission").map(\.path) == ["spec.permissions.filesystem.write[0]", "spec.permissions.filesystem.write[1]", "spec.permissions.network[4]"])695        #expect(d.with("broad-permission").allSatisfy { $0.severity == .warning })696        #expect(d.with("missing-parameter").map(\.path) == ["spec.permissions.filesystem.write[2]"])697        #expect(d.with("invalid-permission-expression").map(\.path) == ["spec.permissions.filesystem.write[3]", "spec.permissions.filesystem.write[4]"])698        #expect(d.with("expression-syntax").map(\.path) == ["spec.permissions.filesystem.write[5]"])699        #expect(d.with("invalid-permission-host").map(\.path) == ["spec.permissions.network[0]", "spec.permissions.network[1]", "spec.permissions.network[2]",700                                                                 "spec.permissions.network[5]", "spec.permissions.browser[1]"])701        #expect(d.with("invalid-value").map(\.path).sorted() == ["spec.permissions.processes[1]", "spec.permissions.scripts[1]"])702    }703704    @Test func permissionSufficiencyForPaths() {705        let d = check("""706          parameters:707            - {name: inbox, type: path, default: ~/Inbox}708            - {name: free, type: path, required: true}709          permissions:710            filesystem:711              read: ["${{ params.inbox }}", "${{ params.free }}", /data/in]712              write: [~/Out]713              delete: [~/Trashable]714          steps:715            - id: readOK716              tool: filesystem.read717              with: {path: "${{ params.inbox }}/sub/file.txt"}718            - id: readFree719              tool: filesystem.list720              with: {path: "${{ params.free }}/deep"}721            - id: readDotDot722              tool: filesystem.read723              with: {path: /data/in/../secret.txt}724            - id: readOutside725              tool: filesystem.read726              with: {path: ~/Elsewhere/file.txt}727            - id: readPrefixTrap728              tool: filesystem.read729              with: {path: ~/Inbox2/file.txt}730            - id: writeOutside731              tool: filesystem.write732              with: {path: ~/Inbox/new.txt, content: x}733            - id: copy734              tool: filesystem.copy735              with: {source: ~/Private/a, destination: ~/Out/a}736            - id: moveOut737              tool: filesystem.move738              with: {source: ~/Inbox/a, destination: ~/Out/a}739            - id: trashOK740              tool: filesystem.trash741              with: {paths: [~/Trashable/a, ~/Inbox/b]}742            - id: dynamic743              tool: filesystem.read744              with: {path: "${{ steps.readOK.output.next }}"}745              needs: [readOK]746            - id: relative747              tool: filesystem.read748              with: {path: notes.txt}749            - id: render750              tool: documents.renderPDF751              with: {source: ~/Inbox/r.md, destination: ~/Desktop/r.pdf}752            - id: shot753              tool: screen.capture754              with: {destination: ~/Out/s.png}755        """)756        let m = d.with("permission-missing")757        #expect(m.allSatisfy { $0.severity == .warning })758        func flagged(_ step: String, _ kind: String) -> Bool { m.contains { $0.stepID == step && $0.message.contains("filesystem.\(kind)") } }759        #expect(!m.contains { ["readOK", "readFree", "dynamic", "relative"].contains($0.stepID ?? "") }, "\(m.summary)")760        #expect(flagged("readDotDot", "read") && flagged("readOutside", "read") && flagged("readPrefixTrap", "read"))761        #expect(flagged("writeOutside", "write"))762        #expect(flagged("copy", "read") && !flagged("copy", "write"))763        // Regression: the real filesystem.move needs write (not delete) on the source and write on the destination.764        #expect(flagged("moveOut", "write") && !flagged("moveOut", "read") && !flagged("moveOut", "delete"))765        #expect(m.filter { $0.stepID == "moveOut" }.map(\.path) == ["spec.steps[7].with.source"])766        #expect(m.filter { $0.stepID == "trashOK" }.map(\.path) == ["spec.steps[8].with.paths[1]"])767        #expect(flagged("render", "write") && !flagged("render", "read"))768        #expect(m.contains { $0.stepID == "shot" && $0.message.contains("permissions.screen") } && !flagged("shot", "write"))769    }770771    @Test func permissionSufficiencyForHostsProcessesRuntimesAndTheRest() {772        let spec = """773          parameters:774            - {name: site, type: string, default: "https://api.example.com/v1"}775          PERMISSIONS776          steps:777            - id: http778              tool: http.request779              with: {url: "${{ params.site }}/items"}780            - id: httpOther781              tool: http.request782              with: {url: "https://other.test/x"}783            - id: browse784              tool: browser.open785              with: {url: "https://shop.example.org/cart"}786            - id: run787              tool: shell.run788              with: {executable: /usr/bin/make, args: [build, install]}789            - id: bash790              tool: shell.script791              with: {script: "echo hi"}792            - id: py793              tool: python.run794              with: {script: "print(1)"}795            - id: osa796              tool: applescript.run797              with: {script: "return 1"}798            - id: git799              tool: git.status800              with: {path: /repo}801            - id: clipIn802              tool: clipboard.read803            - id: clipOut804              tool: clipboard.write805              with: {content: x}806            - id: post807              tool: notifications.post808              with: {title: T, body: B}809            - id: tell810              notify: Hello811            - id: shot812              tool: screen.capture813              with: {destination: /out/s.png}814            - id: launch815              tool: system.launchApp816              with: {bundleID: com.apple.Safari}817            - id: press818              tool: accessibility.press819              with: {bundleID: com.apple.Safari, selector: OK}820            - id: key821              tool: http.request822              with:823                url: https://api.example.com/login824                headers: {Authorization: "${{ secret('api-token') }}"}825        """826        let bare = check(spec.replacingOccurrences(of: "PERMISSIONS", with: "permissions: {filesystem: {read: [/repo], write: [/out]}}"))827        let m = bare.with("permission-missing")828        #expect(m.allSatisfy { $0.severity == .warning })829        for (step, text) in [("http", "api.example.com"), ("httpOther", "other.test"), ("browse", "permissions.browser"), ("run", "'make'"),830                             ("bash", "'bash'"), ("py", "'python'"), ("osa", "'applescript'"), ("git", "runs git"), ("clipIn", "clipboard.read"),831                             ("clipOut", "clipboard.write"), ("post", "permissions.notifications"), ("tell", "permissions.notifications"),832                             ("shot", "permissions.screen"), ("launch", "app control"), ("press", "accessibility control"), ("key", "api-token")] {833            #expect(m.contains { $0.stepID == step && $0.message.contains(text) }, "expected a permission warning for \(step) mentioning \(text)\n\(m.summary)")834        }835        let full = check(spec.replacingOccurrences(of: "PERMISSIONS", with: """836        permissions:837            filesystem: {read: [/repo], write: [/out]}838            processes: [make, git]839            scripts: [bash, python, applescript]840            apps: {accessibility: [com.apple.Safari], control: [com.apple.Safari]}841            network: ["*.example.com", other.test]842            browser: [shop.example.org]843            secrets: [api-token]844            clipboard: {read: true, write: true}845            notifications: true846            screen: [display]847        """))848        #expect(full.with("permission-missing").isEmpty, "\(full.summary)")849    }850851    @Test func budgetRules() {852        let bad = check("""853          budget: {maxDurationSeconds: 0, maxProviderCostUSD: -1, maxToolCalls: 0, maxTokens: 0}854          steps:855            - {id: a, delay: 1}856        """)857        #expect(bad.with("invalid-budget").map(\.path) == ["spec.budget.maxDurationSeconds", "spec.budget.maxProviderCostUSD", "spec.budget.maxToolCalls", "spec.budget.maxTokens"])858        let warn = check("""859          budget: {maxDurationSeconds: 60, maxToolCalls: 1}860          permissions: {filesystem: {read: [/x]}, notifications: true}861          steps:862            - id: slow863              tool: filesystem.list864              with: {path: /x}865              timeoutSeconds: 120866            - id: again867              tool: filesystem.list868              with: {path: /x}869            - id: ag870              agent: {instructions: Think}871              objective: Think872        """)873        #expect(warn.has("budget-missing-cost-limit", .warning))874        #expect(warn.has("budget-timeout-exceeds-duration", .warning, step: "slow"))875        #expect(warn.has("budget-too-small", .warning))876        let unused = check("""877          budget: {maxProviderCostUSD: 2}878          steps:879            - {id: a, delay: 1}880        """)881        #expect(unused.has("budget-unused", .info) && unused.errors.isEmpty)882    }883884    @Test func unsupportedVersionAndKindAreReportedOnHandBuiltDocuments() {885        var doc = WorkflowDocument(apiVersion: "orix.ai/v9", kind: "Job", metadata: WorkflowMetadata(id: "x", name: "X"))886        doc.spec.steps = [WorkflowStep(id: "a", kind: .delay(DelayStep(seconds: 1)))]887        let d = WorkflowValidator.validate(doc, tools: [], agents: [], workflows: [])888        #expect(d.has("unsupported-version") && d.has("unsupported-kind"))889    }890}891892@Suite("Declared capabilities, proposed grants and source positions (unit)") struct ValidatorCapabilityTests {893    private func document() throws -> WorkflowDocument {894        try WorkflowCodec.parse("""895        apiVersion: orix.ai/v1896        kind: Automation897        metadata: {id: caps, name: Caps}898        spec:899          parameters:900            - {name: inbox, type: path, default: ~/Inbox}901            - {name: needed, type: path, required: true}902          permissions:903            filesystem:904              read: ["~/Docs/../Docs/In", "${{ params.inbox }}", "${{ params.needed }}", relative, "~bob/x", "~"]905              write: [/var/tmp/out]906              delete: [~/Docs/In]907            processes: [git]908            scripts: [python]909            apps: {automation: [com.apple.iCal], accessibility: [com.apple.Safari], control: [com.apple.Preview]}910            network: [API.Example.com]911            browser: ["*.example.org"]912            secrets: [smtp-password]913            clipboard: {read: true}914            notifications: true915            screen: ["window:com.apple.Safari"]916          steps:917            - {id: a, delay: 1}918        """)919    }920921    @Test func declaredCapabilitiesExpandTildeAgainstTheGivenHome() throws {922        let caps = try document().declaredCapabilities(home: "/Users/tester")923        #expect(caps == [924            .fileRead(path: "/Users/tester/Docs/In"), .fileRead(path: "/Users/tester/Inbox"), .fileRead(path: "/Users/tester"),925            .fileWrite(path: PathResolver.resolveSymlinks("/var/tmp/out")), .fileDelete(path: "/Users/tester/Docs/In"),926            .processExec(executable: "git"), .scriptRuntime(name: "python"), .appAutomation(bundleID: "com.apple.iCal"),927            .accessibility(bundleID: "com.apple.Safari"), .appControl(bundleID: "com.apple.Preview"), .network(host: "api.example.com"),928            .browser(host: "*.example.org"), .secret(reference: "smtp-password"), .clipboardRead, .notify,929            .screenCapture(scope: "window:com.apple.Safari"),930        ])931        let supplied = try document().declaredCapabilities(home: "/Users/tester", parameters: ["needed": "~/Needed", "inbox": "/abs/inbox"])932        #expect(supplied.contains(.fileRead(path: "/Users/tester/Needed")) && supplied.contains(.fileRead(path: "/abs/inbox")))933        #expect(!supplied.contains(.fileRead(path: "/Users/tester/Inbox")))934    }935936    @Test func proposedGrantsAreWorkflowScopedAndNeverAboveTierTwo() throws {937        let now = Date(timeIntervalSince1970: 1_800_000_000)938        let grants = try document().proposedGrants(workflowID: "wf-9", home: "/Users/tester", now: now)939        #expect(grants.count == 16)940        #expect(grants.allSatisfy { $0.scope == .workflow(workflowID: "wf-9") && $0.maxTier <= .consequential && $0.source == "workflow:wf-9" && $0.createdAt == now && $0.isActive(at: now) })941        func tier(_ c: Capability) -> ApprovalTier? { grants.first { $0.capability == c }?.maxTier }942        // Regression: a read grant must be usable as the input of write and consequential calls (copy, move, run in a943        // folder); the policy engine compares the grant ceiling with the tier of the whole call.944        #expect(tier(.fileRead(path: "/Users/tester/Inbox")) == .consequential)945        #expect(tier(.fileDelete(path: "/Users/tester/Docs/In")) == .reversibleWrite)946        #expect(tier(.clipboardRead) == .reversibleWrite && tier(.notify) == .reversibleWrite)947        #expect(tier(.screenCapture(scope: "window:com.apple.Safari")) == .reversibleWrite)948        #expect(tier(.fileWrite(path: "/Users/tester/Out")) ?? tier(.fileWrite(path: "/Users/tester/Docs/Out")) != .consequential)949        #expect(tier(.processExec(executable: "git")) == .consequential && tier(.network(host: "api.example.com")) == .consequential)950        #expect(Set(grants.map(\.id)).count == grants.count)951        #expect(WorkflowDocument.proposedTier(for: .secret(reference: "x")) == .consequential)952    }953954    @Test func parameterDefaultsFeedPermissionResolution() throws {955        let doc = try document()956        #expect(doc.parameterDefaults == ["inbox": "~/Inbox"])957        #expect(doc.resolvePermissionEntry("${{ params.inbox }}/a", parameters: [:]) == "~/Inbox/a")958        #expect(doc.resolvePermissionEntry("${{ params.needed }}", parameters: [:]) == nil)959        #expect(doc.resolvePermissionEntry("${{ env.HOME }}", parameters: [:]) == nil)960        #expect(doc.resolvePermissionEntry("plain", parameters: [:]) == "plain")961    }962963    @Test func diagnoseAttachesSourcePositionsToValidatorDiagnostics() {964        let source = """965        apiVersion: orix.ai/v1966        kind: Automation967        metadata: {id: pos, name: Pos}968        spec:969          steps:970            - id: a971              tool: filesystem.lst972              with:973                path: /x974            - id: b975              needs: [ghost]976              delay: 1977        """978        let result = WorkflowCodec.diagnose(source: source, tools: CanonicalTools.manifests())979        #expect(result.document != nil)980        let tool = result.diagnostics.first { $0.code == "unknown-tool" }981        #expect(tool?.line == 7 && tool?.column == 7)982        let need = result.diagnostics.first { $0.code == "unknown-need" }983        #expect(need?.line == 11 && need?.column == 15, "\(need?.line ?? 0):\(need?.column ?? 0)")984        let broken = WorkflowCodec.diagnose(source: "spec: [", tools: [])985        #expect(broken.document == nil && broken.diagnostics.first?.code == "syntax" && broken.diagnostics.first?.line == 1)986    }987}988
Files main
Menu

Theme