Repository Public License
1// ORIX OS Automation2// File: Packages/ORIXWorkflowEngine/Tests/ORIXWorkflowEngineTests/ValidatorTests.swift3// Description: Unit tests of the static validator: one focused case per diagnostic code, plus capabilities, proposed grants and source positions.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Testing9import ORIXCore10@testable import ORIXWorkflowEngine1112/// Builds a document from the body of `spec:` (written at two-space indentation) and validates it.13private func check(_ spec: String, tools: [ToolManifest]? = nil, agents: [String] = [], workflows: [String] = [],14 sourceLocation: SourceLocation = #_sourceLocation) -> [WorkflowDiagnostic] {15 let source = "apiVersion: orix.ai/v1\nkind: Automation\nmetadata:\n id: t\n name: T\nspec:\n" + spec16 do {17 let doc = try WorkflowCodec.parse(source)18 return WorkflowValidator.validate(doc, tools: tools ?? validatorTools, agents: agents, workflows: workflows)19 } catch {20 Issue.record("test source does not parse: \(error.localizedDescription)", sourceLocation: sourceLocation)21 return []22 }23}2425/// The canonical catalogue plus one tool with a strict input schema.26private let validatorTools: [ToolManifest] = CanonicalTools.manifests() + [27 ToolManifest(identifier: "test.strict", name: "Strict", summary: "Tool with a strict input schema for validator tests.", family: "test",28 inputSchema: .object(["path": .string(), "count": .integer(minimum: 1), "tags": .array(of: .string())], required: ["path"]),29 sideEffect: .read, defaultTier: .readOnly, idempotent: true),30]3132private extension Array where Element == WorkflowDiagnostic {33 func with(_ code: String) -> [WorkflowDiagnostic] { filter { $0.code == code } }34 func has(_ code: String, _ severity: WorkflowDiagnostic.Severity = .error, step: String? = nil, containing text: String = "") -> Bool {35 contains { $0.code == code && $0.severity == severity && (step == nil || $0.stepID == step) && (text.isEmpty || $0.message.contains(text)) }36 }37 var errors: [WorkflowDiagnostic] { filter { $0.severity == .error } }38 var summary: String { map(\.description).joined(separator: "\n") }39}4041@Suite("Workflow validator: identifiers and graph (unit)") struct ValidatorGraphTests {42 @Test func cleanDocumentHasNoDiagnostics() {43 let d = check("""44 steps:45 - id: a46 delay: 147 - id: b48 needs: [a]49 transform: ${{ steps.a.status }}50 """)51 #expect(d.isEmpty, "\(d.summary)")52 }5354 @Test func emptyWorkflowIsAWarning() {55 #expect(check(" steps: []\n").has("empty-workflow", .warning))56 }5758 @Test func invalidIdentifiers() {59 let d = check("""60 steps:61 - id: 1bad62 delay: 163 - id: "has space"64 delay: 165 - id: ok-id_266 delay: 167 """)68 #expect(d.with("invalid-id").map(\.stepID) == ["1bad", "has space"])69 #expect(d.with("invalid-id").first?.path == "spec.steps[0].id")70 var doc = WorkflowDocument(metadata: WorkflowMetadata(id: "bad id", name: " "))71 doc.spec.steps = [WorkflowStep(id: "a", kind: .delay(DelayStep(seconds: 1)))]72 let m = WorkflowValidator.validate(doc, tools: [], agents: [], workflows: [])73 #expect(m.has("invalid-id", containing: "metadata.id") && m.has("missing-field", containing: "metadata.name"))74 #expect(WorkflowValidator.isValidIdentifier("a-b_1") && !WorkflowValidator.isValidIdentifier("-a") && !WorkflowValidator.isValidIdentifier(""))75 }7677 @Test func duplicateIdsAreDocumentWideIncludingNestedSteps() {78 let d = check("""79 steps:80 - id: a81 delay: 182 - id: loop83 foreach: [1, 2]84 steps:85 - id: a86 delay: 187 """)88 #expect(d.has("duplicate-id", step: "a", containing: "spec.steps[0]"))89 #expect(d.with("duplicate-id").first?.path == "spec.steps[1].steps[0].id")90 }9192 @Test func selfNeedAndUnknownNeed() {93 let d = check("""94 steps:95 - id: a96 needs: [a, ghost]97 delay: 198 - id: box99 foreach: [1]100 steps:101 - id: inner102 delay: 1103 - id: b104 needs: [inner]105 delay: 1106 """)107 #expect(d.has("self-need", step: "a"))108 #expect(d.has("unknown-need", step: "a", containing: "'ghost'"))109 #expect(d.has("unknown-need", step: "b", containing: "another scope"))110 #expect(d.with("unknown-need").first?.path == "spec.steps[0].needs[1]")111 }112113 @Test func unknownBranchStep() {114 let d = check("""115 steps:116 - id: c117 if: "true"118 then: [ghost]119 else: [c]120 - id: s121 switch: "1"122 cases:123 - value: 1124 steps: [nowhere]125 """)126 #expect(d.with("unknown-branch-step").count == 3)127 #expect(d.has("unknown-branch-step", step: "s", containing: "'nowhere'"))128 }129130 @Test func dependencyCycleReportsItsPath() {131 let d = check("""132 steps:133 - id: a134 needs: [b]135 delay: 1136 - id: b137 needs: [c]138 delay: 1139 - id: c140 needs: [a]141 delay: 1142 - id: free143 delay: 1144 """)145 let cycles = d.with("dependency-cycle")146 #expect(cycles.count == 1)147 #expect(cycles.first?.message.contains("c → b → a → c") == true, "\(cycles.first?.message ?? "")")148 #expect(cycles.first?.path == "spec.steps[0].needs" && cycles.first?.stepID == "a")149 }150151 @Test func cycleThroughAConditionGateIsDetected() {152 let d = check("""153 steps:154 - id: gate155 needs: [target]156 if: "true"157 then: [target]158 - id: target159 delay: 1160 """)161 #expect(d.has("dependency-cycle"))162 }163164 @Test func structuralErrorsOnlyReportGraphProblems() throws {165 let doc = try WorkflowCodec.parse("apiVersion: orix.ai/v1\nkind: Automation\nmetadata: {id: t, name: T}\nspec:\n steps:\n - id: a\n needs: [zz]\n tool: nope.tool\n")166 #expect(WorkflowValidator.structuralErrors(doc).map(\.code) == ["unknown-need"])167 }168169 @Test func unreachableSteps() {170 let d = check("""171 steps:172 - id: work173 delay: 1174 onError: handler175 - id: handler176 delay: 1177 - id: afterHandler178 needs: [handler]179 delay: 1180 - id: stop181 fail: Always182 - id: afterFail183 needs: [stop]184 delay: 1185 - id: later186 needs: [afterFail]187 delay: 1188 - id: gate189 if: "false"190 then: [never]191 else: [always]192 - id: never193 delay: 1194 - id: always195 delay: 1196 """)197 #expect(d.has("unreachable-step", .error, step: "afterHandler", containing: "onError handler"))198 #expect(d.has("unreachable-step", .warning, step: "afterFail", containing: "unconditional fail step 'stop'"))199 #expect(d.has("unreachable-step", .warning, step: "later"))200 #expect(d.has("unreachable-step", .warning, step: "never", containing: "always false"))201 #expect(!d.with("unreachable-step").contains { $0.stepID == "always" || $0.stepID == "handler" || $0.stepID == "work" })202 let handled = check("""203 steps:204 - id: stop205 fail: Sometimes206 continueOnError: true207 - id: after208 needs: [stop]209 delay: 1210 """)211 #expect(handled.with("unreachable-step").isEmpty)212 }213214 @Test func invalidHandlers() {215 let d = check("""216 steps:217 - id: a218 delay: 1219 onError: a220 - id: b221 delay: 1222 onError: ghost223 - id: c224 delay: 1225 onError: needy226 - id: needy227 needs: [a]228 delay: 1229 - id: d230 delay: 1231 onError: gated232 - id: gate233 if: "true"234 then: [gated]235 - id: gated236 delay: 1237 onError: last238 - id: last239 delay: 1240 """)241 let h = d.with("invalid-handler")242 #expect(h.contains { $0.stepID == "a" && $0.message.contains("its own") })243 #expect(h.contains { $0.stepID == "b" && $0.message.contains("'ghost'") })244 #expect(h.contains { $0.stepID == "c" && $0.message.contains("must not declare 'needs'") })245 #expect(h.contains { $0.stepID == "d" && $0.message.contains("branch") })246 #expect(h.contains { $0.stepID == "d" && $0.severity == .warning && $0.message.contains("not chained") })247 }248249 @Test func staleLayoutEntry() {250 let d = check("""251 steps:252 - id: a253 delay: 1254 layout:255 a: {x: 1, y: 2}256 gone: {x: 3, y: 4}257 "trigger:0": {x: 0, y: 0}258 """)259 #expect(d.with("stale-layout").map(\.path) == ["spec.layout.gone"])260 }261}262263@Suite("Workflow validator: tools, arguments and step rules (unit)") struct ValidatorStepTests {264 @Test func unknownToolNamesItsFamily() {265 let d = check("""266 steps:267 - id: a268 tool: filesystem.lst269 - id: b270 tool: document.renderPDF271 """)272 #expect(d.has("unknown-tool", step: "a", containing: "filesystem.list"))273 #expect(d.has("unknown-tool", step: "b"))274 #expect(d.with("unknown-tool").first?.path == "spec.steps[0].tool")275 }276277 @Test func literalArgumentsAreCheckedAgainstTheInputSchema() {278 let d = check("""279 steps:280 - id: missing281 tool: test.strict282 with: {count: 2}283 - id: unexpected284 tool: test.strict285 with: {path: /x, colour: red}286 - id: wrongType287 tool: test.strict288 with: {path: 12, count: 0, tags: [a, 3]}289 - id: dynamic290 tool: test.strict291 with:292 path: ${{ params.p }}293 count: ${{ params.n }}294 tags: ["${{ params.p }}", 4]295 parameters:296 - {name: p, type: string, default: /x}297 - {name: n, type: integer, default: 1}298 """)299 let a = d.with("invalid-argument")300 #expect(a.contains { $0.stepID == "missing" && $0.message.contains("requires argument 'path'") })301 #expect(a.contains { $0.stepID == "unexpected" && $0.message.contains("no argument 'colour'") && $0.path == "spec.steps[1].with.colour" })302 #expect(a.contains { $0.stepID == "wrongType" && $0.message.contains("'path'") && $0.message.contains("expected string") })303 #expect(a.contains { $0.stepID == "wrongType" && $0.message.contains("'count'") && $0.message.contains("minimum") })304 #expect(a.contains { $0.stepID == "wrongType" && $0.message.contains("'tags.1'") })305 #expect(!a.contains { $0.stepID == "dynamic" }, "values containing ${{ }} are only known at run time")306 }307308 @Test func unboundedLoops() {309 let d = check("""310 steps:311 - id: zero312 while: ${{ vars.go }}313 maxIterations: 0314 steps: [{id: z1, delay: 1}]315 - id: huge316 while: ${{ vars.go }}317 maxIterations: 50000318 steps: [{id: h1, delay: 1}]319 - id: constant320 while: "true"321 maxIterations: 3322 steps: [{id: c1, delay: 1}]323 variables: {go: true}324 """)325 #expect(d.has("unbounded-loop", .error, step: "zero"))326 #expect(d.has("unbounded-loop", .warning, step: "huge", containing: "effectively unbounded"))327 #expect(d.has("unbounded-loop", .warning, step: "constant", containing: "constant"))328 }329330 @Test func invalidValuesOnSteps() {331 let d = check("""332 steps:333 - id: t334 delay: -1335 timeoutSeconds: 0336 retry: {maxAttempts: 0, backoffSeconds: -1}337 - id: fe338 foreach: [1]339 as: steps340 concurrency: 0341 collect: ghost342 steps: []343 - id: st344 set: {"bad name": 1}345 - id: tr346 transform: "1"347 into: 9x348 - id: empty349 set: {}350 - id: v351 verify: []352 - id: par353 parallel:354 - name: same355 steps: []356 - name: same357 steps: [{id: p1, delay: 1}]358 """)359 #expect(d.with("invalid-value").filter { $0.stepID == "t" }.count == 4)360 #expect(d.has("invalid-value", step: "fe", containing: "concurrency"))361 #expect(d.has("invalid-value", step: "fe", containing: "'as: steps'"))362 #expect(d.has("unknown-step-reference", step: "fe", containing: "'collect'"))363 #expect(d.has("empty-body", .warning, step: "fe") && d.has("empty-body", .warning, step: "empty") && d.has("empty-body", .warning, step: "par"))364 #expect(d.has("invalid-id", step: "st") && d.has("invalid-id", step: "tr"))365 #expect(d.has("empty-verify", .warning, step: "v"))366 #expect(d.has("duplicate-branch", step: "par"))367 }368369 @Test func conditionAndSwitchWarnings() {370 let d = check("""371 steps:372 - id: c373 if: ${{ vars.x }}374 - id: both375 if: ${{ vars.x }}376 then: [t]377 else: [t]378 - id: t379 delay: 1380 - id: s381 switch: ${{ vars.x }}382 cases:383 - {value: 1, steps: [u]}384 - {value: 1, steps: []}385 - id: u386 delay: 1387 - id: s2388 switch: ${{ vars.x }}389 variables: {x: 1}390 """)391 #expect(d.has("empty-branches", .warning, step: "c") && d.has("empty-branches", .warning, step: "s2"))392 #expect(d.has("ambiguous-branch", .warning, step: "both"))393 #expect(d.has("duplicate-case", .warning, step: "s"))394 }395396 @Test func retryOnNonIdempotentStepsIsFlagged() {397 let d = check("""398 steps:399 - id: read400 tool: filesystem.list401 with: {path: /tmp/x}402 retry: 3403 - id: move404 tool: filesystem.move405 with: {source: /tmp/x/a, destination: /tmp/x/b}406 retry: {maxAttempts: 3}407 - id: forced408 tool: filesystem.move409 with: {source: /tmp/x/a, destination: /tmp/x/b}410 retry: {maxAttempts: 3, force: true}411 - id: ag412 agent: {instructions: Do it}413 objective: Something414 retry: 2415 - id: box416 foreach: [1]417 retry: 2418 steps: [{id: b1, delay: 1}]419 permissions:420 filesystem: {read: [/tmp/x], write: [/tmp/x], delete: [/tmp/x]}421 budget: {maxProviderCostUSD: 1}422 """)423 let r = d.with("retry-not-idempotent")424 #expect(r.map(\.stepID) == ["move", "ag", "box"], "\(r.summary)")425 #expect(r.allSatisfy { $0.severity == .warning } && r[0].message.contains("'filesystem.move' is not idempotent"))426 }427428 @Test func agentRules() {429 let d = check("""430 steps:431 - id: named432 agent: research-analyst433 objective: Research434 - id: unknown435 agent: nobody436 objective: Research437 - id: inline438 agent: {instructions: " ", model: sonnet, tools: [filesystem.*, nope.*, "*"]}439 expect:440 - fileExists: ~/a441 - truthy: "true"442 budget: {maxProviderCostUSD: 1}443 """, agents: ["research-analyst"])444 #expect(d.with("unknown-agent").map(\.stepID) == ["unknown"])445 #expect(d.has("missing-field", step: "inline", containing: "instructions"))446 #expect(d.has("invalid-model", step: "inline", containing: "provider/model"))447 #expect(d.with("unknown-tool").map(\.path) == ["spec.steps[2].agent.tools[1]"])448 #expect(d.has("missing-objective", .warning, step: "inline") && !d.has("missing-objective", .warning, step: "named"))449 #expect(d.with("invalid-assertion").map(\.path) == ["spec.steps[2].expect[1]"])450 }451452 @Test func subworkflowRules() {453 let d = check("""454 steps:455 - id: ok456 subworkflow: shared-cleanup457 - id: missing458 subworkflow: nowhere459 - id: selfByID460 subworkflow: t461 - id: selfByName462 subworkflow: T463 """, workflows: ["shared-cleanup"])464 #expect(d.with("unknown-workflow").map(\.stepID) == ["missing"])465 #expect(d.with("recursive-subworkflow").map(\.stepID) == ["selfByID", "selfByName"])466 }467}468469@Suite("Workflow validator: expressions (unit)") struct ValidatorExpressionTests {470 @Test func syntaxErrorsAndUnknownFunctions() {471 let d = check("""472 steps:473 - id: a474 when: ${{ 1 + }}475 delay: 1476 - id: b477 transform: nope(1) + len("x")478 - id: c479 tool: filesystem.list480 with: {path: "/tmp/${{ unterminated"}481 - id: d482 if: " "483 then: [a]484 """)485 #expect(d.has("expression-syntax", step: "a") && d.with("expression-syntax").first?.path == "spec.steps[0].when")486 #expect(d.has("unknown-function", step: "b", containing: "'nope'") && d.with("unknown-function").count == 1)487 #expect(d.has("expression-syntax", step: "c", containing: "unterminated"))488 #expect(d.has("expression-syntax", step: "d", containing: "empty expression"))489 }490491 @Test func parametersVariablesEnvironmentAndNames() {492 let d = check("""493 parameters:494 - {name: declared, type: string, default: x}495 variables: {known: 1}496 steps:497 - id: a498 set:499 later: ${{ params.declared + params.missing }}500 - id: b501 needs: [a]502 transform: ${{ vars.known + vars.later + vars.never }}503 into: fromTransform504 - id: c505 needs: [b]506 transform: ${{ vars.fromTransform }}507 - id: d508 transform: ${{ env.HOME + env.PATH + env["SHELL"] }}509 - id: e510 transform: ${{ item + stranger + map([1], x => x + other) }}511 """)512 #expect(d.with("missing-parameter").count == 1 && d.has("missing-parameter", step: "a", containing: "'missing'"))513 #expect(d.with("unset-variable").count == 1 && d.has("unset-variable", .warning, step: "b", containing: "vars.never"))514 #expect(d.with("env-not-allowed").map { $0.message.contains("env.PATH") || $0.message.contains("env.SHELL") } == [true, true])515 let names = d.with("unknown-name")516 #expect(names.count == 3 && names.allSatisfy { $0.stepID == "e" })517 #expect(!names.contains { $0.message.contains("'x'") }, "lambda parameters are bound")518 }519520 @Test func stepReferencesMustBeUpstream() {521 let d = check("""522 steps:523 - id: first524 delay: 1525 - id: sibling526 transform: ${{ steps.first.status }}527 - id: ghostRef528 transform: ${{ steps.ghost.output }}529 - id: selfRef530 transform: ${{ steps.selfRef.output }}531 - id: box532 needs: [first]533 foreach: [1]534 steps:535 - id: inner536 transform: ${{ steps.first.status }}537 - id: outside538 needs: [box]539 transform: ${{ steps.inner.output }}540 - id: v541 verify:542 - stepSucceeded: first543 - stepSucceeded: ghost544 """)545 let up = d.with("step-not-upstream")546 #expect(up.contains { $0.stepID == "sibling" && $0.message.contains("add it to 'needs'") })547 #expect(up.contains { $0.stepID == "selfRef" && $0.message.contains("its own result") })548 #expect(up.contains { $0.stepID == "outside" && $0.message.contains("container's output") })549 #expect(up.contains { $0.stepID == "v" && $0.path == "spec.steps[6].verify[0].stepSucceeded" })550 #expect(!up.contains { $0.stepID == "inner" }, "a nested step sees what its container sees")551 #expect(d.with("unknown-step-reference").map(\.stepID) == ["ghostRef", "v"])552 }553554 @Test func allowedReferencesProduceNoDiagnostic() {555 let d = check("""556 variables: {n: 0}557 steps:558 - id: a559 delay: 1560 - id: b561 needs: [a]562 delay: 1563 - id: transitive564 needs: [b]565 transform: ${{ steps.a.status + steps.b.status }}566 - id: gate567 needs: [a]568 if: ${{ steps.a.status == "succeeded" }}569 then: [gated]570 - id: gated571 transform: ${{ steps.gate.output.result && steps.a.status == "succeeded" }}572 - id: each573 needs: [a]574 foreach: [1, 2]575 as: thing576 collect: use577 steps:578 - id: use579 transform: ${{ thing + index + len(steps.a.status) }}580 - id: again581 while: ${{ loop.index < 3 && steps.bump.output != 2 && default(steps.peek.output, 0) < 5 }}582 maxIterations: 5583 steps:584 - id: bump585 transform: ${{ loop.index + default(steps.peek.output, 0) }}586 - id: peek587 transform: ${{ loop.index }}588 - id: risky589 delay: 1590 onError: recover591 - id: recover592 transform: ${{ error.message + steps.risky.status }}593 onError:594 - id: report595 transform: ${{ error.stepID + steps.a.status + steps.transitive.status }}596 """)597 #expect(d.isEmpty, "\(d.summary)")598 }599600 @Test func loopVariablesAreScoped() {601 let d = check("""602 steps:603 - id: each604 foreach: [1]605 steps:606 - id: inside607 transform: ${{ item }}608 - id: outside609 needs: [each]610 transform: ${{ item }}611 - id: noError612 transform: ${{ error.message }}613 - id: noLoop614 transform: ${{ loop.index }}615 """)616 #expect(d.with("unknown-name").map(\.stepID) == ["outside", "noError", "noLoop"])617 }618}619620@Suite("Workflow validator: triggers, parameters, permissions, budget (unit)") struct ValidatorSpecTests {621 @Test(arguments: [622 ("0 9 * * MON", true), ("*/15 * * * *", true), ("0 9-17 * * MON-FRI", true), ("5,35 0 1 jan,jul *", true), ("0 0 * * 7", true),623 ("0 9 * *", false), ("60 * * * *", false), ("* 24 * * *", false), ("0 0 0 * *", false), ("0 0 * 13 *", false),624 ("0 0 * * FUNDAY", false), ("*/0 * * * *", false), ("10-5 * * * *", false), ("0 9 * * MON extra", false), ("a b c d e", false),625 ])626 func cronExpressions(cron: String, valid: Bool) {627 #expect((WorkflowValidator.cronProblem(cron) == nil) == valid, "\(cron): \(WorkflowValidator.cronProblem(cron) ?? "ok")")628 }629630 @Test func invalidTriggers() {631 let d = check("""632 triggers:633 - {type: schedule, cron: "61 * * * *", timezone: America/Toronto}634 - {type: schedule, cron: "0 9 * * *", timezone: Mars/Olympus}635 - {type: once, at: tomorrow}636 - {type: once, at: "2026-12-01T09:00:00-05:00"}637 - {type: interval, seconds: 0}638 - {type: interval, seconds: 5}639 - {type: fileChange, path: relative/dir, debounceSeconds: -1}640 - {type: appLaunch, bundleID: Safari}641 - {type: fileChange, path: ~/ok}642 steps:643 - {id: a, delay: 1}644 """)645 #expect(d.with("invalid-cron").map(\.path) == ["spec.triggers[0].cron"])646 #expect(d.with("invalid-timezone").map(\.path) == ["spec.triggers[1].timezone"])647 let t = d.with("invalid-trigger")648 #expect(t.map(\.path) == ["spec.triggers[2].at", "spec.triggers[4].seconds", "spec.triggers[5].seconds", "spec.triggers[6].path",649 "spec.triggers[6].debounceSeconds", "spec.triggers[7].bundleID"], "\(t.summary)")650 #expect(t[2].severity == .warning && t[1].severity == .error)651 }652653 @Test func parameterRules() {654 let d = check("""655 parameters:656 - {name: a, type: integer, default: 1}657 - {name: a, type: string}658 - {name: bad-name, type: string}659 - {name: n, type: number, default: nope}660 - {name: i, type: integer, default: 1.5}661 - {name: p, type: path, default: 3}662 - {name: ok, type: array, default: [1]}663 - {name: nul, type: object, default: null}664 - {name: req, type: boolean, default: true, required: true}665 steps:666 - {id: a, delay: 1}667 """)668 #expect(d.with("duplicate-parameter").map(\.path) == ["spec.parameters[1].name"])669 #expect(d.has("invalid-id", containing: "'bad-name'"))670 #expect(d.with("parameter-default-type").map(\.path) == ["spec.parameters[3].default", "spec.parameters[4].default", "spec.parameters[5].default"])671 #expect(d.has("parameter-required-default", .info))672 }673674 @Test func permissionEntriesMustBeAbsolutePathsAndBareHosts() {675 let d = check("""676 parameters:677 - {name: folder, type: path, default: ~/Work}678 - {name: rel, type: path, default: work/here}679 permissions:680 filesystem:681 read: [relative/path, "~bob/Documents", "", "${{ params.folder }}/in", "${{ params.rel }}"]682 write: [/, "~", "${{ params.ghost }}", "${{ vars.x }}/y", "${{ steps.a.output }}", "${{ 1 + }}"]683 network: ["https://example.com", "example.com/path", "example.com:8080", "*.example.com", "*", ""]684 browser: [ok.example.com, "bad host"]685 scripts: [bash, ruby]686 processes: [git, ""]687 steps:688 - {id: a, delay: 1}689 """)690 let paths = d.with("invalid-permission-path")691 #expect(paths.map(\.path) == ["spec.permissions.filesystem.read[0]", "spec.permissions.filesystem.read[1]", "spec.permissions.filesystem.read[2]",692 "spec.permissions.filesystem.read[4]"], "\(paths.summary)")693 #expect(paths[0].message.contains("relative") && paths[1].message.contains("~user") && paths[2].message.contains("empty"))694 #expect(d.with("broad-permission").map(\.path) == ["spec.permissions.filesystem.write[0]", "spec.permissions.filesystem.write[1]", "spec.permissions.network[4]"])695 #expect(d.with("broad-permission").allSatisfy { $0.severity == .warning })696 #expect(d.with("missing-parameter").map(\.path) == ["spec.permissions.filesystem.write[2]"])697 #expect(d.with("invalid-permission-expression").map(\.path) == ["spec.permissions.filesystem.write[3]", "spec.permissions.filesystem.write[4]"])698 #expect(d.with("expression-syntax").map(\.path) == ["spec.permissions.filesystem.write[5]"])699 #expect(d.with("invalid-permission-host").map(\.path) == ["spec.permissions.network[0]", "spec.permissions.network[1]", "spec.permissions.network[2]",700 "spec.permissions.network[5]", "spec.permissions.browser[1]"])701 #expect(d.with("invalid-value").map(\.path).sorted() == ["spec.permissions.processes[1]", "spec.permissions.scripts[1]"])702 }703704 @Test func permissionSufficiencyForPaths() {705 let d = check("""706 parameters:707 - {name: inbox, type: path, default: ~/Inbox}708 - {name: free, type: path, required: true}709 permissions:710 filesystem:711 read: ["${{ params.inbox }}", "${{ params.free }}", /data/in]712 write: [~/Out]713 delete: [~/Trashable]714 steps:715 - id: readOK716 tool: filesystem.read717 with: {path: "${{ params.inbox }}/sub/file.txt"}718 - id: readFree719 tool: filesystem.list720 with: {path: "${{ params.free }}/deep"}721 - id: readDotDot722 tool: filesystem.read723 with: {path: /data/in/../secret.txt}724 - id: readOutside725 tool: filesystem.read726 with: {path: ~/Elsewhere/file.txt}727 - id: readPrefixTrap728 tool: filesystem.read729 with: {path: ~/Inbox2/file.txt}730 - id: writeOutside731 tool: filesystem.write732 with: {path: ~/Inbox/new.txt, content: x}733 - id: copy734 tool: filesystem.copy735 with: {source: ~/Private/a, destination: ~/Out/a}736 - id: moveOut737 tool: filesystem.move738 with: {source: ~/Inbox/a, destination: ~/Out/a}739 - id: trashOK740 tool: filesystem.trash741 with: {paths: [~/Trashable/a, ~/Inbox/b]}742 - id: dynamic743 tool: filesystem.read744 with: {path: "${{ steps.readOK.output.next }}"}745 needs: [readOK]746 - id: relative747 tool: filesystem.read748 with: {path: notes.txt}749 - id: render750 tool: documents.renderPDF751 with: {source: ~/Inbox/r.md, destination: ~/Desktop/r.pdf}752 - id: shot753 tool: screen.capture754 with: {destination: ~/Out/s.png}755 """)756 let m = d.with("permission-missing")757 #expect(m.allSatisfy { $0.severity == .warning })758 func flagged(_ step: String, _ kind: String) -> Bool { m.contains { $0.stepID == step && $0.message.contains("filesystem.\(kind)") } }759 #expect(!m.contains { ["readOK", "readFree", "dynamic", "relative"].contains($0.stepID ?? "") }, "\(m.summary)")760 #expect(flagged("readDotDot", "read") && flagged("readOutside", "read") && flagged("readPrefixTrap", "read"))761 #expect(flagged("writeOutside", "write"))762 #expect(flagged("copy", "read") && !flagged("copy", "write"))763 // Regression: the real filesystem.move needs write (not delete) on the source and write on the destination.764 #expect(flagged("moveOut", "write") && !flagged("moveOut", "read") && !flagged("moveOut", "delete"))765 #expect(m.filter { $0.stepID == "moveOut" }.map(\.path) == ["spec.steps[7].with.source"])766 #expect(m.filter { $0.stepID == "trashOK" }.map(\.path) == ["spec.steps[8].with.paths[1]"])767 #expect(flagged("render", "write") && !flagged("render", "read"))768 #expect(m.contains { $0.stepID == "shot" && $0.message.contains("permissions.screen") } && !flagged("shot", "write"))769 }770771 @Test func permissionSufficiencyForHostsProcessesRuntimesAndTheRest() {772 let spec = """773 parameters:774 - {name: site, type: string, default: "https://api.example.com/v1"}775 PERMISSIONS776 steps:777 - id: http778 tool: http.request779 with: {url: "${{ params.site }}/items"}780 - id: httpOther781 tool: http.request782 with: {url: "https://other.test/x"}783 - id: browse784 tool: browser.open785 with: {url: "https://shop.example.org/cart"}786 - id: run787 tool: shell.run788 with: {executable: /usr/bin/make, args: [build, install]}789 - id: bash790 tool: shell.script791 with: {script: "echo hi"}792 - id: py793 tool: python.run794 with: {script: "print(1)"}795 - id: osa796 tool: applescript.run797 with: {script: "return 1"}798 - id: git799 tool: git.status800 with: {path: /repo}801 - id: clipIn802 tool: clipboard.read803 - id: clipOut804 tool: clipboard.write805 with: {content: x}806 - id: post807 tool: notifications.post808 with: {title: T, body: B}809 - id: tell810 notify: Hello811 - id: shot812 tool: screen.capture813 with: {destination: /out/s.png}814 - id: launch815 tool: system.launchApp816 with: {bundleID: com.apple.Safari}817 - id: press818 tool: accessibility.press819 with: {bundleID: com.apple.Safari, selector: OK}820 - id: key821 tool: http.request822 with:823 url: https://api.example.com/login824 headers: {Authorization: "${{ secret('api-token') }}"}825 """826 let bare = check(spec.replacingOccurrences(of: "PERMISSIONS", with: "permissions: {filesystem: {read: [/repo], write: [/out]}}"))827 let m = bare.with("permission-missing")828 #expect(m.allSatisfy { $0.severity == .warning })829 for (step, text) in [("http", "api.example.com"), ("httpOther", "other.test"), ("browse", "permissions.browser"), ("run", "'make'"),830 ("bash", "'bash'"), ("py", "'python'"), ("osa", "'applescript'"), ("git", "runs git"), ("clipIn", "clipboard.read"),831 ("clipOut", "clipboard.write"), ("post", "permissions.notifications"), ("tell", "permissions.notifications"),832 ("shot", "permissions.screen"), ("launch", "app control"), ("press", "accessibility control"), ("key", "api-token")] {833 #expect(m.contains { $0.stepID == step && $0.message.contains(text) }, "expected a permission warning for \(step) mentioning \(text)\n\(m.summary)")834 }835 let full = check(spec.replacingOccurrences(of: "PERMISSIONS", with: """836 permissions:837 filesystem: {read: [/repo], write: [/out]}838 processes: [make, git]839 scripts: [bash, python, applescript]840 apps: {accessibility: [com.apple.Safari], control: [com.apple.Safari]}841 network: ["*.example.com", other.test]842 browser: [shop.example.org]843 secrets: [api-token]844 clipboard: {read: true, write: true}845 notifications: true846 screen: [display]847 """))848 #expect(full.with("permission-missing").isEmpty, "\(full.summary)")849 }850851 @Test func budgetRules() {852 let bad = check("""853 budget: {maxDurationSeconds: 0, maxProviderCostUSD: -1, maxToolCalls: 0, maxTokens: 0}854 steps:855 - {id: a, delay: 1}856 """)857 #expect(bad.with("invalid-budget").map(\.path) == ["spec.budget.maxDurationSeconds", "spec.budget.maxProviderCostUSD", "spec.budget.maxToolCalls", "spec.budget.maxTokens"])858 let warn = check("""859 budget: {maxDurationSeconds: 60, maxToolCalls: 1}860 permissions: {filesystem: {read: [/x]}, notifications: true}861 steps:862 - id: slow863 tool: filesystem.list864 with: {path: /x}865 timeoutSeconds: 120866 - id: again867 tool: filesystem.list868 with: {path: /x}869 - id: ag870 agent: {instructions: Think}871 objective: Think872 """)873 #expect(warn.has("budget-missing-cost-limit", .warning))874 #expect(warn.has("budget-timeout-exceeds-duration", .warning, step: "slow"))875 #expect(warn.has("budget-too-small", .warning))876 let unused = check("""877 budget: {maxProviderCostUSD: 2}878 steps:879 - {id: a, delay: 1}880 """)881 #expect(unused.has("budget-unused", .info) && unused.errors.isEmpty)882 }883884 @Test func unsupportedVersionAndKindAreReportedOnHandBuiltDocuments() {885 var doc = WorkflowDocument(apiVersion: "orix.ai/v9", kind: "Job", metadata: WorkflowMetadata(id: "x", name: "X"))886 doc.spec.steps = [WorkflowStep(id: "a", kind: .delay(DelayStep(seconds: 1)))]887 let d = WorkflowValidator.validate(doc, tools: [], agents: [], workflows: [])888 #expect(d.has("unsupported-version") && d.has("unsupported-kind"))889 }890}891892@Suite("Declared capabilities, proposed grants and source positions (unit)") struct ValidatorCapabilityTests {893 private func document() throws -> WorkflowDocument {894 try WorkflowCodec.parse("""895 apiVersion: orix.ai/v1896 kind: Automation897 metadata: {id: caps, name: Caps}898 spec:899 parameters:900 - {name: inbox, type: path, default: ~/Inbox}901 - {name: needed, type: path, required: true}902 permissions:903 filesystem:904 read: ["~/Docs/../Docs/In", "${{ params.inbox }}", "${{ params.needed }}", relative, "~bob/x", "~"]905 write: [/var/tmp/out]906 delete: [~/Docs/In]907 processes: [git]908 scripts: [python]909 apps: {automation: [com.apple.iCal], accessibility: [com.apple.Safari], control: [com.apple.Preview]}910 network: [API.Example.com]911 browser: ["*.example.org"]912 secrets: [smtp-password]913 clipboard: {read: true}914 notifications: true915 screen: ["window:com.apple.Safari"]916 steps:917 - {id: a, delay: 1}918 """)919 }920921 @Test func declaredCapabilitiesExpandTildeAgainstTheGivenHome() throws {922 let caps = try document().declaredCapabilities(home: "/Users/tester")923 #expect(caps == [924 .fileRead(path: "/Users/tester/Docs/In"), .fileRead(path: "/Users/tester/Inbox"), .fileRead(path: "/Users/tester"),925 .fileWrite(path: PathResolver.resolveSymlinks("/var/tmp/out")), .fileDelete(path: "/Users/tester/Docs/In"),926 .processExec(executable: "git"), .scriptRuntime(name: "python"), .appAutomation(bundleID: "com.apple.iCal"),927 .accessibility(bundleID: "com.apple.Safari"), .appControl(bundleID: "com.apple.Preview"), .network(host: "api.example.com"),928 .browser(host: "*.example.org"), .secret(reference: "smtp-password"), .clipboardRead, .notify,929 .screenCapture(scope: "window:com.apple.Safari"),930 ])931 let supplied = try document().declaredCapabilities(home: "/Users/tester", parameters: ["needed": "~/Needed", "inbox": "/abs/inbox"])932 #expect(supplied.contains(.fileRead(path: "/Users/tester/Needed")) && supplied.contains(.fileRead(path: "/abs/inbox")))933 #expect(!supplied.contains(.fileRead(path: "/Users/tester/Inbox")))934 }935936 @Test func proposedGrantsAreWorkflowScopedAndNeverAboveTierTwo() throws {937 let now = Date(timeIntervalSince1970: 1_800_000_000)938 let grants = try document().proposedGrants(workflowID: "wf-9", home: "/Users/tester", now: now)939 #expect(grants.count == 16)940 #expect(grants.allSatisfy { $0.scope == .workflow(workflowID: "wf-9") && $0.maxTier <= .consequential && $0.source == "workflow:wf-9" && $0.createdAt == now && $0.isActive(at: now) })941 func tier(_ c: Capability) -> ApprovalTier? { grants.first { $0.capability == c }?.maxTier }942 // Regression: a read grant must be usable as the input of write and consequential calls (copy, move, run in a943 // folder); the policy engine compares the grant ceiling with the tier of the whole call.944 #expect(tier(.fileRead(path: "/Users/tester/Inbox")) == .consequential)945 #expect(tier(.fileDelete(path: "/Users/tester/Docs/In")) == .reversibleWrite)946 #expect(tier(.clipboardRead) == .reversibleWrite && tier(.notify) == .reversibleWrite)947 #expect(tier(.screenCapture(scope: "window:com.apple.Safari")) == .reversibleWrite)948 #expect(tier(.fileWrite(path: "/Users/tester/Out")) ?? tier(.fileWrite(path: "/Users/tester/Docs/Out")) != .consequential)949 #expect(tier(.processExec(executable: "git")) == .consequential && tier(.network(host: "api.example.com")) == .consequential)950 #expect(Set(grants.map(\.id)).count == grants.count)951 #expect(WorkflowDocument.proposedTier(for: .secret(reference: "x")) == .consequential)952 }953954 @Test func parameterDefaultsFeedPermissionResolution() throws {955 let doc = try document()956 #expect(doc.parameterDefaults == ["inbox": "~/Inbox"])957 #expect(doc.resolvePermissionEntry("${{ params.inbox }}/a", parameters: [:]) == "~/Inbox/a")958 #expect(doc.resolvePermissionEntry("${{ params.needed }}", parameters: [:]) == nil)959 #expect(doc.resolvePermissionEntry("${{ env.HOME }}", parameters: [:]) == nil)960 #expect(doc.resolvePermissionEntry("plain", parameters: [:]) == "plain")961 }962963 @Test func diagnoseAttachesSourcePositionsToValidatorDiagnostics() {964 let source = """965 apiVersion: orix.ai/v1966 kind: Automation967 metadata: {id: pos, name: Pos}968 spec:969 steps:970 - id: a971 tool: filesystem.lst972 with:973 path: /x974 - id: b975 needs: [ghost]976 delay: 1977 """978 let result = WorkflowCodec.diagnose(source: source, tools: CanonicalTools.manifests())979 #expect(result.document != nil)980 let tool = result.diagnostics.first { $0.code == "unknown-tool" }981 #expect(tool?.line == 7 && tool?.column == 7)982 let need = result.diagnostics.first { $0.code == "unknown-need" }983 #expect(need?.line == 11 && need?.column == 15, "\(need?.line ?? 0):\(need?.column ?? 0)")984 let broken = WorkflowCodec.diagnose(source: "spec: [", tools: [])985 #expect(broken.document == nil && broken.diagnostics.first?.code == "syntax" && broken.diagnostics.first?.line == 1)986 }987}988