Repository Public License
1// ORIX OS Automation2// File: Packages/ORIXWorkflowEngine/Tests/ORIXWorkflowEngineTests/EngineTests.swift3// Description: Workflow engine tests with a closure-backed tool invoker and a fake agent runner (mocked tools), plus real-file verification in temp directories.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Testing9import ORIXCore10import ORIXTestingSupport11@testable import ORIXWorkflowEngine1213// MARK: - Test doubles1415/// Wraps the shared closure invoker and marks chosen tools as supporting dry runs (the shared double cannot).16struct DryRunAwareInvoker: ToolInvoking {17 let base: ClosureToolInvoker18 let dryRunTools: Set<String>19 func manifests() async -> [ToolManifest] {20 await base.manifests().map { m in var m = m; m.supportsDryRun = dryRunTools.contains(m.identifier); return m }21 }22 func invoke(_ call: ToolCall, context: InvocationContext) async -> ToolResult { await base.invoke(call, context: context) }23}2425actor FakeAgentRunner: AgentRunning {26 private(set) var tasks: [AgentTask] = []27 private(set) var profiles: [AgentProfile] = []28 private let reply: @Sendable (AgentTask) -> AgentOutcome29 init(reply: @escaping @Sendable (AgentTask) -> AgentOutcome) { self.reply = reply }30 func run(task: AgentTask, profile: AgentProfile, runID: String, workflowID: String?, stepID: String?) async -> AgentOutcome {31 tasks.append(task); profiles.append(profile)32 return reply(task)33 }34}3536/// Records when each call ran, to prove overlap (or its absence) with timestamps.37actor Timeline {38 private(set) var spans: [String: (start: Date, end: Date)] = [:]39 private var active = 040 private(set) var maxActive = 041 private(set) var canceled: [String] = []42 func enter() -> Date { active += 1; maxActive = max(maxActive, active); return Date() }43 func leave(_ id: String, _ start: Date) { active -= 1; spans[id] = (start, Date()) }44 func markCanceled(_ id: String) { canceled.append(id) }45 func overlap(_ a: String, _ b: String) -> Bool {46 guard let x = spans[a], let y = spans[b] else { return false }47 return x.start < y.end && y.start < x.end48 }49}5051actor Counter {52 private(set) var value = 053 func next() -> Int { value += 1; return value }54}5556struct Harness {57 let tmp: TempDirectory58 let tools = ClosureToolInvoker()59 let sink = CollectingEventSink()60 let store = InMemoryCheckpointStore(keepHistory: true)6162 init() throws { tmp = try TempDirectory(prefix: "orix-wf") }6364 func engine(invoker: (any ToolInvoking)? = nil, agents: (any AgentRunning)? = nil, loader: (any SubworkflowLoading)? = nil,65 approval: WorkflowApprovalHandler? = nil, profiles: AgentProfileResolver? = nil, clock: any WorkflowClock = SystemWorkflowClock(),66 configuration: WorkflowEngineConfiguration = .init()) -> WorkflowEngine {67 WorkflowEngine(tools: invoker ?? tools, agents: agents, events: sink, checkpoints: store, loader: loader, approval: approval,68 agentProfiles: profiles, clock: clock, configuration: configuration)69 }7071 func options(_ parameters: [String: JSONValue] = [:], dryRun: Bool = false, breakpoints: Set<String> = [], startPaused: Bool = false,72 resolutions: [String: InFlightResolution] = [:]) -> WorkflowRunOptions {73 WorkflowRunOptions(runID: "run-1", parameters: parameters, dryRun: dryRun, workingDirectory: tmp.url,74 artifactsDirectory: tmp.url.appendingPathComponent("artifacts"), breakpoints: breakpoints, startPaused: startPaused,75 inFlightResolutions: resolutions)76 }7778 /// Registers `test.echo` (returns its arguments), `test.fail` and `test.effect` (not idempotent).79 func registerBasics() async {80 await tools.register("test.echo") { call, _ in .success(call.arguments, summary: "echo") }81 await tools.register("test.fail") { _, _ in .failure(.failed, ToolFailure(code: "io", message: "disk on fire")) }82 await tools.register("test.effect", sideEffect: .write, tier: .reversibleWrite, idempotent: false) { call, _ in .success(call.arguments, summary: "effect") }83 await tools.register("notifications.post", sideEffect: .write, tier: .reversibleWrite, idempotent: false) { call, _ in .success(call.arguments, summary: "posted") }84 }8586 func calls(_ toolID: String) async -> [ToolCall] { await tools.calls.filter { $0.toolID == toolID } }87}8889func workflow(_ steps: String, spec: String = "") throws -> WorkflowDocument {90 try WorkflowCodec.parse("apiVersion: orix.ai/v1\nkind: Automation\nmetadata:\n id: t\n name: Test\nspec:\n" + spec + " steps:\n" + steps)91}9293func waitUntil(timeout: Double = 5, _ condition: @Sendable () async -> Bool) async -> Bool {94 let deadline = Date().addingTimeInterval(timeout)95 while Date() < deadline {96 if await condition() { return true }97 try? await Task.sleep(nanoseconds: 2_000_000)98 }99 return await condition()100}101102func sleepingTool(_ timeline: Timeline, seconds: Double) -> ClosureToolInvoker.Handler {103 { call, _ in104 let id = call.arguments["id"]?.stringValue ?? call.id105 let start = await timeline.enter()106 do { try await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000)) }107 catch {108 await timeline.leave(id, start)109 await timeline.markCanceled(id)110 return .failure(.canceled, ToolFailure(code: "canceled", message: "Canceled"))111 }112 await timeline.leave(id, start)113 return .success(["id": .string(id)], summary: "slept")114 }115}116117// MARK: - Scheduling118119@Suite("Engine scheduling (mocked tools)") struct EngineSchedulingTests {120 @Test func stepsRunInDependencyOrderAndPassOutputs() async throws {121 let h = try Harness(); defer { h.tmp.remove() }122 await h.registerBasics()123 let doc = try workflow("""124 - id: c125 needs: [b]126 tool: test.echo127 with: {text: "${{ steps.b.output.text }}-c", status: "${{ steps.a.status }}"}128 - id: a129 tool: test.echo130 with: {text: a}131 - id: b132 needs: [a]133 tool: test.echo134 with: {text: "${{ steps.a.output.text }}-b"}135 - id: out136 needs: [c]137 output: {final: "${{ steps.c.output.text }}", count: "${{ len(steps.c.output.text) }}"}138139 """)140 let outcome = await h.engine().run(doc, options: h.options())141 #expect(outcome.status == .succeeded)142 #expect(outcome.steps.map(\.stepID) == ["a", "b", "c", "out"])143 #expect(outcome.outputs == ["final": "a-b-c", "count": 5])144 #expect(outcome.step("c")?.output["status"] == "succeeded")145 #expect(outcome.usage.toolCalls == 3)146 let started = await h.sink.events(of: .stepStarted).map(\.stepID)147 #expect(started == ["a", "b", "c", "out"])148 let finished = await h.sink.events(of: .stepFinished)149 #expect(finished.count == 4 && finished.allSatisfy { $0.data["status"] == "succeeded" })150 #expect(await h.sink.events(of: .checkpoint).count == 4)151 #expect(outcome.steps.allSatisfy { $0.attempts == 1 && $0.startedAt != nil && $0.endedAt != nil })152 }153154 @Test func independentStepsOverlapInTime() async throws {155 let h = try Harness(); defer { h.tmp.remove() }156 let timeline = Timeline()157 await h.tools.register("test.sleep", handler: sleepingTool(timeline, seconds: 0.25))158 let doc = try workflow("""159 - id: left160 tool: test.sleep161 with: {id: left}162 - id: right163 tool: test.sleep164 with: {id: right}165 - id: join166 needs: [left, right]167 tool: test.sleep168 with: {id: join}169170 """)171 let began = Date()172 let outcome = await h.engine().run(doc, options: h.options())173 let elapsed = Date().timeIntervalSince(began)174 #expect(outcome.status == .succeeded)175 #expect(await timeline.overlap("left", "right"), "independent steps must run at the same time")176 let leftJoin = await timeline.overlap("left", "join"), rightJoin = await timeline.overlap("right", "join")177 #expect(!leftJoin && !rightJoin)178 #expect(await timeline.maxActive == 2)179 #expect(elapsed < 0.74, "three 0.25 s steps took \(elapsed) s; two of them must have overlapped")180 }181182 @Test func maxConcurrentStepsBoundsParallelism() async throws {183 let h = try Harness(); defer { h.tmp.remove() }184 let timeline = Timeline()185 await h.tools.register("test.sleep", handler: sleepingTool(timeline, seconds: 0.05))186 let doc = try workflow((1...5).map { " - id: s\($0)\n tool: test.sleep\n with: {id: s\($0)}\n" }.joined())187 let outcome = await h.engine(configuration: .init(maxConcurrentSteps: 2)).run(doc, options: h.options())188 #expect(outcome.status == .succeeded)189 #expect(await timeline.maxActive == 2)190 let serial = Timeline()191 let h2 = try Harness(); defer { h2.tmp.remove() }192 await h2.tools.register("test.sleep", handler: sleepingTool(serial, seconds: 0.03))193 _ = await h2.engine(configuration: .init(maxConcurrentSteps: 1)).run(doc, options: h2.options())194 #expect(await serial.maxActive == 1)195 }196197 @Test func conditionRoutesBranchesAndSkipsPropagate() async throws {198 let h = try Harness(); defer { h.tmp.remove() }199 await h.registerBasics()200 let doc = try workflow("""201 - id: check202 if: ${{ params.n > 2 }}203 then: [big]204 else: [small]205 - id: big206 tool: test.echo207 with: {size: big}208 - id: small209 tool: test.echo210 with: {size: small}211 - id: afterSmall212 needs: [small]213 tool: test.echo214 - id: gated215 when: ${{ params.n == 0 }}216 tool: test.echo217 - id: always218 needs: [check]219 tool: test.echo220221 """, spec: " parameters:\n - {name: n, type: integer, default: 1}\n")222 let big = await h.engine().run(doc, options: h.options(["n": 5]))223 #expect(big.status == .succeeded)224 #expect(big.step("check")?.output == ["result": true, "branch": "then"])225 #expect(big.step("big")?.status == .succeeded)226 #expect(big.step("small")?.status == .skipped && big.step("small")?.note?.contains("'then' branch") == true)227 #expect(big.step("afterSmall")?.status == .skipped && big.step("afterSmall")?.note?.contains("'small' was skipped") == true)228 #expect(big.step("gated")?.status == .skipped && big.step("gated")?.note?.contains("when") == true)229 #expect(big.step("always")?.status == .succeeded)230 #expect(await h.calls("test.echo").count == 2)231232 let h2 = try Harness(); defer { h2.tmp.remove() }233 await h2.registerBasics()234 let small = await h2.engine().run(doc, options: h2.options())235 #expect(small.step("big")?.status == .skipped && small.step("small")?.status == .succeeded && small.step("afterSmall")?.status == .succeeded)236 }237238 @Test func switchSelectsCaseOrDefault() async throws {239 let doc = try workflow("""240 - id: pick241 switch: ${{ params.mode }}242 cases:243 - value: fast244 steps: [quick]245 - value: 2246 steps: [two]247 default: [fallback]248 - id: quick249 tool: test.echo250 - id: two251 tool: test.echo252 - id: fallback253 tool: test.echo254255 """, spec: " parameters:\n - {name: mode, type: string, default: fast}\n")256 for (mode, expected) in [("fast", "quick"), ("other", "fallback")] {257 let h = try Harness(); defer { h.tmp.remove() }258 await h.registerBasics()259 let outcome = await h.engine().run(doc, options: h.options(["mode": .string(mode)]))260 #expect(outcome.status == .succeeded)261 for id in ["quick", "two", "fallback"] { #expect(outcome.step(id)?.status == (id == expected ? .succeeded : .skipped), "\(mode): \(id)") }262 }263 }264265 @Test func foreachFansOutInOrderWithBoundedConcurrency() async throws {266 let h = try Harness(); defer { h.tmp.remove() }267 let timeline = Timeline()268 await h.tools.register("test.sleep", handler: sleepingTool(timeline, seconds: 0.03))269 await h.registerBasics()270 let doc = try workflow("""271 - id: each272 foreach: ${{ params.names }}273 as: name274 concurrency: 3275 collect: shout276 steps:277 - id: nap278 tool: test.sleep279 with: {id: "${{ name }}"}280 - id: shout281 needs: [nap]282 transform: ${{ upper(name) + string(index) }}283 - id: all284 needs: [each]285 output: {names: "${{ steps.each.output.results }}", count: "${{ steps.each.output.count }}"}286287 """, spec: " parameters:\n - {name: names, type: array, default: [a, b, c, d, e, f, g]}\n")288 let outcome = await h.engine(configuration: .init(maxConcurrentSteps: 8)).run(doc, options: h.options())289 #expect(outcome.status == .succeeded)290 #expect(outcome.outputs["names"] == ["A0", "B1", "C2", "D3", "E4", "F5", "G6"])291 #expect(outcome.outputs["count"] == 7)292 #expect(await timeline.maxActive == 3, "foreach concurrency bounds the fan-out")293 #expect(outcome.results(for: "shout").map(\.key).sorted().first == "each[0]/shout")294 #expect(outcome.results(for: "nap").count == 7)295 }296297 @Test func parallelBranchesRunTogetherAndJoin() async throws {298 let h = try Harness(); defer { h.tmp.remove() }299 let timeline = Timeline()300 await h.tools.register("test.sleep", handler: sleepingTool(timeline, seconds: 0.15))301 let doc = try workflow("""302 - id: both303 parallel:304 - name: left305 steps:306 - id: l307 tool: test.sleep308 with: {id: l}309 - name: right310 steps:311 - id: r312 tool: test.sleep313 with: {id: r}314 - id: after315 needs: [both]316 output: {left: "${{ steps.both.output.left.l.id }}", right: "${{ steps.both.output.right.r.id }}"}317318 """)319 let outcome = await h.engine().run(doc, options: h.options())320 #expect(outcome.status == .succeeded && outcome.outputs == ["left": "l", "right": "r"])321 #expect(await timeline.overlap("l", "r"))322 }323324 @Test func loopRunsUntilConditionIsFalseAndStopsAtItsBound() async throws {325 let h = try Harness(); defer { h.tmp.remove() }326 await h.registerBasics()327 let doc = try workflow("""328 - id: poll329 while: ${{ vars.n < params.target }}330 maxIterations: 5331 steps:332 - id: bump333 set: {n: "${{ vars.n + 1 }}", seen: "${{ loop.index }}"}334 - id: done335 needs: [poll]336 output: {n: "${{ vars.n }}", iterations: "${{ steps.poll.output.iterations }}"}337338 """, spec: " parameters:\n - {name: target, type: integer, default: 3}\n variables:\n n: 0\n")339 let ok = await h.engine().run(doc, options: h.options())340 #expect(ok.status == .succeeded && ok.outputs == ["n": 3, "iterations": 3])341 #expect(ok.results(for: "bump").map(\.key) == ["poll#0/bump", "poll#1/bump", "poll#2/bump"])342343 let h2 = try Harness(); defer { h2.tmp.remove() }344 let bounded = await h2.engine().run(doc, options: h2.options(["target": 99]))345 #expect(bounded.status == .failed)346 #expect(bounded.step("poll")?.errorCode == "loop_bound" && bounded.results(for: "bump").count == 5)347 #expect(bounded.step("done") == nil, "nothing runs after an unhandled failure")348 }349350 @Test func foreachRefusesNonLists() async throws {351 let h = try Harness(); defer { h.tmp.remove() }352 let doc = try workflow(" - id: each\n foreach: ${{ 42 }}\n steps:\n - id: x\n delay: 0\n")353 let outcome = await h.engine().run(doc, options: h.options())354 #expect(outcome.status == .failed && outcome.step("each")?.error?.contains("needs a list") == true)355 }356}357358// MARK: - Failure handling359360@Suite("Engine retries, timeouts and error routing (mocked tools)") struct EngineFailureTests {361 static let flakyDoc = """362 - id: flaky363 tool: TOOL364 retry: {maxAttempts: 4, backoffSeconds: 1, backoffMultiplier: 2FORCE}365366 """367368 func runFlaky(tool: String, idempotent: Bool, force: Bool) async throws -> (WorkflowOutcome, [Double], [RunEvent], Int) {369 let h = try Harness(); defer { h.tmp.remove() }370 let counter = Counter()371 await h.tools.register(tool, sideEffect: idempotent ? .read : .write, tier: idempotent ? .readOnly : .reversibleWrite, idempotent: idempotent) { _, _ in372 let n = await counter.next()373 return n < 3 ? .failure(.failed, ToolFailure(code: "network", message: "attempt \(n) failed", retryable: true)) : .success(["attempt": .int(n)], summary: "ok")374 }375 let clock = VirtualWorkflowClock()376 let doc = try workflow(Self.flakyDoc.replacingOccurrences(of: "TOOL", with: tool).replacingOccurrences(of: "FORCE", with: force ? ", force: true" : ""))377 let outcome = await h.engine(clock: clock).run(doc, options: h.options())378 return (outcome, clock.sleeps, await h.sink.events(of: .retry), await counter.value)379 }380381 @Test func idempotentToolIsRetriedWithBackoff() async throws {382 let (outcome, sleeps, events, calls) = try await runFlaky(tool: "test.flakyRead", idempotent: true, force: false)383 #expect(outcome.status == .succeeded)384 #expect(outcome.step("flaky")?.attempts == 3 && calls == 3)385 #expect(sleeps == [1, 2], "exponential backoff through the injected clock")386 #expect(events.count == 2 && events.allSatisfy { $0.data["retried"] == true && $0.stepID == "flaky" })387 }388389 @Test func nonIdempotentToolIsNotBlindlyRetriedAndTheEventSaysWhy() async throws {390 let (outcome, sleeps, events, calls) = try await runFlaky(tool: "test.flakyWrite", idempotent: false, force: false)391 #expect(outcome.status == .failed)392 #expect(outcome.step("flaky")?.attempts == 1 && calls == 1 && sleeps.isEmpty)393 #expect(events.count == 1 && events[0].data["retried"] == false)394 #expect(events[0].text.contains("not idempotent") && events[0].text.contains("retry.force"))395 }396397 @Test func forceRetriesANonIdempotentTool() async throws {398 let (outcome, _, events, calls) = try await runFlaky(tool: "test.flakyWrite", idempotent: false, force: true)399 #expect(outcome.status == .succeeded && calls == 3)400 #expect(events.count == 2 && events.allSatisfy { $0.data["forced"] == true })401 }402403 @Test func retriesStopAtMaxAttempts() async throws {404 let h = try Harness(); defer { h.tmp.remove() }405 await h.registerBasics()406 let doc = try workflow(" - id: f\n tool: test.fail\n retry: {maxAttempts: 3}\n")407 let outcome = await h.engine().run(doc, options: h.options())408 #expect(outcome.status == .failed && outcome.step("f")?.attempts == 3)409 #expect(await h.calls("test.fail").count == 3)410 #expect(outcome.errors.contains { $0.contains("disk on fire") })411 }412413 @Test func stepTimeoutCancelsTheToolAndMapsToTimedOut() async throws {414 let h = try Harness(); defer { h.tmp.remove() }415 let timeline = Timeline()416 await h.tools.register("test.sleep", handler: sleepingTool(timeline, seconds: 10))417 await h.registerBasics()418 let doc = try workflow("""419 - id: slow420 tool: test.sleep421 with: {id: slow}422 timeoutSeconds: 0.1423 - id: after424 needs: [slow]425 tool: test.echo426427 """)428 let began = Date()429 let outcome = await h.engine().run(doc, options: h.options())430 #expect(Date().timeIntervalSince(began) < 5)431 #expect(outcome.status == .timedOut)432 #expect(outcome.step("slow")?.status == .timedOut && outcome.step("slow")?.errorCode == "timeout")433 #expect(await timeline.canceled == ["slow"], "the timeout must cancel the running tool")434 #expect(outcome.step("after") == nil)435436 let h2 = try Harness(); defer { h2.tmp.remove() }437 await h2.tools.register("test.sleep", handler: sleepingTool(Timeline(), seconds: 10))438 await h2.registerBasics()439 var tolerant = doc440 tolerant.spec.steps[0].continueOnError = true441 let partial = await h2.engine().run(tolerant, options: h2.options())442 #expect(partial.status == .partiallySucceeded && partial.step("after")?.status == .succeeded)443 }444445 @Test func continueOnErrorYieldsPartialSuccess() async throws {446 let h = try Harness(); defer { h.tmp.remove() }447 await h.registerBasics()448 let doc = try workflow("""449 - id: bad450 tool: test.fail451 continueOnError: true452 - id: next453 needs: [bad]454 tool: test.echo455 with: {saw: "${{ steps.bad.status }}", why: "${{ steps.bad.error }}"}456457 """)458 let outcome = await h.engine().run(doc, options: h.options())459 #expect(outcome.status == .partiallySucceeded)460 #expect(outcome.step("bad")?.status == .failed && outcome.step("bad")?.continued == true)461 #expect(outcome.step("next")?.output == ["saw": "failed", "why": "disk on fire"])462 }463464 @Test func onErrorRoutesToACompensationStep() async throws {465 let h = try Harness(); defer { h.tmp.remove() }466 await h.registerBasics()467 let doc = try workflow("""468 - id: write469 tool: test.fail470 onError: undo471 - id: undo472 tool: test.echo473 with: {compensating: "${{ error.stepID }}", because: "${{ error.message }}", status: "${{ steps.write.status }}"}474 - id: next475 needs: [write]476 tool: test.echo477 with: {handled: yes}478479 """)480 let outcome = await h.engine().run(doc, options: h.options())481 #expect(outcome.status == .partiallySucceeded)482 #expect(outcome.step("write")?.handledBy == "undo" && outcome.step("write")?.continued == true)483 #expect(outcome.step("undo")?.output == ["compensating": "write", "because": "disk on fire", "status": "failed"])484 #expect(outcome.step("next")?.status == .succeeded)485 #expect(outcome.steps.map(\.stepID) == ["undo", "write", "next"])486 }487488 @Test func handlerOnlyRunsOnFailureAndAFailingHandlerFailsTheRun() async throws {489 let h = try Harness(); defer { h.tmp.remove() }490 await h.registerBasics()491 let fine = try workflow(" - id: ok\n tool: test.echo\n onError: undo\n - id: undo\n tool: test.echo\n")492 let clean = await h.engine().run(fine, options: h.options())493 #expect(clean.status == .succeeded && clean.step("undo") == nil, "a handler step never runs in the normal flow")494495 let h2 = try Harness(); defer { h2.tmp.remove() }496 await h2.registerBasics()497 let broken = try workflow(" - id: bad\n tool: test.fail\n onError: undo\n - id: undo\n fail: cannot undo\n")498 let failed = await h2.engine().run(broken, options: h2.options())499 #expect(failed.status == .failed && failed.step("bad")?.handledBy == nil && failed.step("undo")?.status == .failed)500 }501502 @Test func workflowLevelOnErrorRunsAfterAFailure() async throws {503 let h = try Harness(); defer { h.tmp.remove() }504 await h.registerBasics()505 let doc = try workflow(" - id: bad\n fail: \"stop: ${{ 1 + 1 }}\"\n",506 spec: " onError:\n - id: alert\n notify: \"${{ error.stepID }} failed: ${{ error.message }}\"\n")507 let outcome = await h.engine().run(doc, options: h.options())508 #expect(outcome.status == .failed)509 #expect(outcome.step("bad")?.error == "stop: 2" && outcome.step("alert")?.status == .succeeded)510 #expect(await h.calls("notifications.post").first?.arguments == ["title": "Test", "body": "bad failed: stop: 2"])511 }512513 @Test func failureInsideAContainerCanBeHandledByTheContainer() async throws {514 let h = try Harness(); defer { h.tmp.remove() }515 await h.registerBasics()516 let doc = try workflow("""517 - id: each518 foreach: [1, 2, 3]519 continueOnError: true520 steps:521 - id: maybe522 when: ${{ item == 2 }}523 tool: test.fail524 - id: after525 needs: [each]526 tool: test.echo527528 """)529 let outcome = await h.engine().run(doc, options: h.options())530 #expect(outcome.status == .partiallySucceeded)531 #expect(outcome.step("each")?.status == .failed && outcome.step("each")?.error?.contains("iteration 1") == true)532 #expect(outcome.step("after")?.status == .succeeded)533 }534535 @Test func expressionErrorsFailTheStepWithADiagnostic() async throws {536 let h = try Harness(); defer { h.tmp.remove() }537 await h.registerBasics()538 let doc = try workflow(" - id: bad\n tool: test.echo\n with: {x: \"${{ 1 / 0 }}\"}\n")539 let outcome = await h.engine().run(doc, options: h.options())540 #expect(outcome.status == .failed && outcome.step("bad")?.errorCode == "expression")541 #expect(outcome.diagnostics.first?.stepID == "bad" && outcome.diagnostics.first?.message.contains("division by zero") == true)542 #expect(await h.tools.calls.isEmpty)543 }544545 @Test func interpolatedToolOutputIsNeverReEvaluated() async throws {546 let h = try Harness(); defer { h.tmp.remove() }547 await h.registerBasics()548 await h.tools.register("test.hostile") { _, _ in .success(["text": "${{ secret(\"api\") }} ${{ env.HOME }}"], summary: "web page") }549 let doc = try workflow("""550 - id: fetch551 tool: test.hostile552 - id: use553 needs: [fetch]554 tool: test.echo555 with: {exact: "${{ steps.fetch.output.text }}", mixed: "got ${{ steps.fetch.output.text }}"}556557 """)558 let outcome = await h.engine().run(doc, options: h.options())559 #expect(outcome.status == .succeeded)560 #expect(outcome.step("use")?.output == ["exact": "${{ secret(\"api\") }} ${{ env.HOME }}", "mixed": "got ${{ secret(\"api\") }} ${{ env.HOME }}"])561 }562}563564// MARK: - Approvals, dry run, budgets, status mapping565566@Suite("Engine approvals, dry run, budgets and statuses (mocked tools)") struct EnginePolicyTests {567 static let gated = """568 - id: prepare569 tool: test.effect570 with: {n: 1}571 - id: gate572 needs: [prepare]573 approval: "Send ${{ steps.prepare.output.n }} file?"574 - id: send575 needs: [gate]576 tool: test.effect577 with: {n: 2}578579 """580581 @Test func approvalApprovedDeniedAndUnanswered() async throws {582 let h = try Harness(); defer { h.tmp.remove() }583 await h.registerBasics()584 let doc = try workflow(Self.gated)585 let approved = await h.engine(approval: { prompt in586 #expect(prompt.message == "Send 1 file?" && prompt.stepID == "gate")587 return .approveOnce588 }).run(doc, options: h.options())589 #expect(approved.status == .succeeded && approved.step("gate")?.output == ["approved": true])590 #expect(await h.sink.events(of: .approvalRequested).count == 1)591 #expect(await h.sink.events(of: .approvalResolved).count == 1)592 let transitions = await h.sink.events(of: .statusChanged).map(\.text)593 #expect(transitions.contains("RUNNING → WAITING_APPROVAL") && transitions.contains("WAITING_APPROVAL → RUNNING"))594595 let h2 = try Harness(); defer { h2.tmp.remove() }596 await h2.registerBasics()597 let denied = await h2.engine(approval: { _ in .deny(reason: "not today") }).run(doc, options: h2.options())598 #expect(denied.status == .failed && denied.step("gate")?.error == "approval denied: not today")599 #expect(await h2.calls("test.effect").count == 1)600601 let h3 = try Harness(); defer { h3.tmp.remove() }602 await h3.registerBasics()603 let unanswered = await h3.engine().run(doc, options: h3.options())604 #expect(unanswered.status == .waitingApproval && unanswered.step("gate")?.status == .notApproved)605 #expect(await h3.calls("test.effect").count == 1, "nothing after the gate may run")606607 // The run can be resumed from its checkpoint once somebody answers: the completed step is not run again.608 let checkpoint = try #require(unanswered.checkpoint)609 let resumed = await h3.engine(approval: { _ in .approveForRun }).resume(from: checkpoint, options: h3.options()).outcome()610 #expect(resumed.status == .succeeded)611 #expect(await h3.calls("test.effect").map { $0.arguments["n"] } == [1, 2])612 }613614 @Test func dryRunExecutesNothingAndSaysExactlyWhatWasSimulated() async throws {615 let h = try Harness(); defer { h.tmp.remove() }616 await h.registerBasics()617 await h.tools.register("test.plan", sideEffect: .write, tier: .reversibleWrite, idempotent: false) { call, _ in618 call.dryRun ? .simulated(["wouldMove": 3], summary: "would move 3 files") : .success(["moved": 3], summary: "moved 3 files")619 }620 let agent = FakeAgentRunner { _ in AgentOutcome(status: .succeeded) }621 let invoker = DryRunAwareInvoker(base: h.tools, dryRunTools: ["test.plan"])622 let doc = try workflow("""623 - id: plan624 tool: test.plan625 with: {path: /tmp/x}626 - id: effect627 needs: [plan]628 tool: test.effect629 - id: think630 needs: [effect]631 agent: {instructions: Think, model: anthropic/test}632 objective: Decide633 - id: gate634 needs: [think]635 approval: Go?636 - id: wait637 needs: [gate]638 delay: 3600639 - id: check640 needs: [wait]641 verify:642 - truthy: ${{ steps.effect.output.done }}643 - id: tell644 needs: [check]645 notify: Finished646 - id: out647 needs: [tell]648 output: {planned: "${{ steps.plan.output.wouldMove }}"}649650 """)651 let began = Date()652 let outcome = await h.engine(invoker: invoker, agents: agent, approval: { _ in Issue.record("no approval in a dry run"); return .approveOnce })653 .run(doc, options: h.options(dryRun: true))654 #expect(Date().timeIntervalSince(began) < 5)655 #expect(outcome.status == .succeeded && outcome.dryRun)656 let calls = await h.tools.calls657 #expect(calls.map(\.toolID) == ["test.plan"], "only dry-run capable tools are invoked")658 #expect(calls.allSatisfy { $0.dryRun }, "every invocation carries dryRun: true")659 #expect(await agent.tasks.isEmpty)660 #expect(outcome.steps.filter { $0.stepID != "out" }.allSatisfy { $0.status == .simulated })661 #expect(outcome.step("plan")?.note == "simulated: test.plan was invoked with dryRun: true — would move 3 files")662 #expect(outcome.step("effect")?.note == "simulated: not executed (test.effect: the tool has no dry-run mode)")663 #expect(outcome.step("think")?.note?.contains("agent was not run") == true)664 #expect(outcome.step("tell")?.note?.hasPrefix("simulated: not executed (notifications.post") == true)665 #expect(outcome.simulations.count == 7 && outcome.simulations[0].hasPrefix("plan: simulated:"))666 #expect(outcome.outputs == ["planned": 3])667 #expect(outcome.verifications.isEmpty)668 }669670 @Test func toolCallBudgetStopsTheRun() async throws {671 let h = try Harness(); defer { h.tmp.remove() }672 await h.registerBasics()673 let doc = try workflow("""674 - id: a675 tool: test.echo676 - id: b677 needs: [a]678 tool: test.echo679 - id: c680 needs: [b]681 tool: test.echo682683 """, spec: " budget:\n maxToolCalls: 2\n")684 let outcome = await h.engine().run(doc, options: h.options())685 #expect(outcome.status == .budgetExceeded)686 #expect(outcome.step("c")?.status == .budgetExceeded && outcome.usage.toolCalls == 2)687 #expect(await h.tools.calls.count == 2)688 #expect(outcome.errors.contains { $0.contains("maxToolCalls") })689 }690691 @Test func agentCostBudgetStopsTheRunAndUsageIsAggregated() async throws {692 let h = try Harness(); defer { h.tmp.remove() }693 await h.registerBasics()694 let agent = FakeAgentRunner { task in695 var usage = RunUsage()696 usage.turns = 2; usage.toolCalls = 1; usage.costUSD = 0.75697 usage.tokens = TokenUsage(inputTokens: 100, outputTokens: 50)698 return AgentOutcome(status: .succeeded, finalText: "done: \(task.objective)", result: ["n": 1], usage: usage)699 }700 let doc = try workflow("""701 - id: first702 agent: writer703 objective: Draft ${{ params.topic }}704 inputs: {topic: "${{ params.topic }}"}705 expect:706 - fileNonEmpty: ~/out/${{ params.topic }}.md707 - id: second708 needs: [first]709 agent: writer710 objective: Refine ${{ steps.first.output.text }}711 - id: third712 needs: [second]713 tool: test.echo714715 """, spec: " parameters:\n - {name: topic, type: string, default: cats}\n budget:\n maxProviderCostUSD: 1\n")716 let profile = AgentProfile(name: "writer", instructions: "Write.", model: ModelRef(provider: .anthropic, model: "test"))717 let outcome = await h.engine(agents: agent, profiles: { $0 == "writer" ? profile : nil }).run(doc, options: h.options())718 #expect(outcome.status == .budgetExceeded)719 #expect(outcome.usage.costUSD == 1.5 && outcome.usage.turns == 4 && outcome.usage.tokens.totalTokens == 300 && outcome.usage.toolCalls == 2)720 #expect(outcome.step("second")?.status == .succeeded && outcome.step("third") == nil)721 let tasks = await agent.tasks722 #expect(tasks.count == 2)723 #expect(tasks[0].objective == "Draft cats" && tasks[0].inputs == ["topic": "cats"])724 #expect(tasks[0].expectations == [Expectation(kind: "file_nonempty", path: "~/out/cats.md")])725 #expect(tasks[0].budget.maxCostUSD == 1 && tasks[1].budget.maxCostUSD == 0.25, "the agent only gets what is left of the workflow budget")726 #expect(tasks[1].objective == "Refine done: Draft cats")727 }728729 @Test func agentStepNeedsARuntimeAndAModel() async throws {730 let h = try Harness(); defer { h.tmp.remove() }731 let doc = try workflow(" - id: a\n agent: {instructions: Hi}\n objective: Go\n")732 let noRuntime = await h.engine().run(doc, options: h.options())733 #expect(noRuntime.status == .failed && noRuntime.step("a")?.error?.contains("no agent runtime") == true)734 let agent = FakeAgentRunner { _ in AgentOutcome(status: .succeeded) }735 let h2 = try Harness(); defer { h2.tmp.remove() }736 let noModel = await h2.engine(agents: agent).run(doc, options: h2.options())737 #expect(noModel.status == .failed && noModel.step("a")?.error?.contains("no default model") == true)738 let h3 = try Harness(); defer { h3.tmp.remove() }739 let ok = await h3.engine(agents: agent, configuration: .init(defaultModel: ModelRef(provider: .openai, model: "m"))).run(doc, options: h3.options())740 #expect(ok.status == .succeeded)741 let inline = await agent.profiles.first742 #expect(inline?.model == ModelRef(provider: .openai, model: "m") && inline?.toolPatterns == [])743 }744745 @Test func durationBudgetMapsToTimedOut() async throws {746 let h = try Harness(); defer { h.tmp.remove() }747 await h.registerBasics()748 let doc = try workflow("""749 - id: wait750 delay: 120751 - id: late752 needs: [wait]753 tool: test.echo754755 """, spec: " budget:\n maxDurationSeconds: 60\n")756 let clock = VirtualWorkflowClock()757 let outcome = await h.engine(clock: clock).run(doc, options: h.options())758 #expect(outcome.status == .timedOut && clock.sleeps == [120])759 #expect(outcome.step("late") == nil && outcome.errors.contains { $0.contains("maxDurationSeconds") })760761 // Real time: the watchdog cancels a running step.762 let h2 = try Harness(); defer { h2.tmp.remove() }763 let timeline = Timeline()764 await h2.tools.register("test.sleep", handler: sleepingTool(timeline, seconds: 10))765 let slow = try workflow(" - id: slow\n tool: test.sleep\n with: {id: slow}\n", spec: " budget:\n maxDurationSeconds: 0.15\n")766 let real = await h2.engine().run(slow, options: h2.options())767 #expect(real.status == .timedOut && real.step("slow")?.status == .canceled)768 #expect(await timeline.canceled == ["slow"])769 }770771 @Test(arguments: [772 (ToolStatus.succeeded, RunStatus.succeeded), (.failed, .failed), (.denied, .failed), (.invalidInput, .failed), (.unsupported, .failed),773 (.notApproved, .waitingApproval), (.blockedPermission, .blockedPermission), (.timedOut, .timedOut), (.canceled, .canceled),774 ])775 func toolStatusMapsToRunStatus(tool: ToolStatus, expected: RunStatus) async throws {776 let h = try Harness(); defer { h.tmp.remove() }777 await h.registerBasics()778 await h.tools.register("test.status") { _, _ in779 tool == .succeeded ? .success(.null, summary: "ok") : .failure(tool, ToolFailure(code: tool.rawValue, message: "tool said \(tool.rawValue)"))780 }781 let doc = try workflow(" - id: s\n tool: test.status\n - id: after\n needs: [s]\n tool: test.echo\n")782 let outcome = await h.engine().run(doc, options: h.options())783 #expect(outcome.status == expected)784 #expect(outcome.step("s")?.status == WorkflowEngine.stepStatus(for: tool))785 #expect((outcome.step("after") != nil) == (tool == .succeeded))786 // Every transition the run announced is legal for the Core state machine.787 var current = RunStatus.queued788 for e in await h.sink.events(of: .statusChanged) {789 let next = try #require(RunStatus(rawValue: e.data["to"]?.stringValue ?? ""))790 #expect(current.canTransition(to: next), "\(current.rawValue) → \(next.rawValue)")791 current = next792 }793 #expect(current == expected)794 }795796 @Test func unhandledStepStatusTable() {797 let expected: [StepStatus: RunStatus] = [798 .failed: .failed, .timedOut: .timedOut, .canceled: .canceled, .notApproved: .waitingApproval,799 .blockedPermission: .blockedPermission, .budgetExceeded: .budgetExceeded, .recoveryRequired: .recoveryRequired,800 ]801 for (step, run) in expected { #expect(WorkflowEngine.runStatus(forUnhandled: step) == run) }802 #expect(StepStatus.simulated.isSuccess && StepStatus.succeeded.isSuccess && !StepStatus.skipped.isSuccess)803 }804805 @Test func invalidDocumentsAndParametersAreRefusedBeforeAnythingRuns() async throws {806 let h = try Harness(); defer { h.tmp.remove() }807 await h.registerBasics()808 let cyclic = try workflow(" - id: a\n needs: [b]\n tool: test.echo\n - id: b\n needs: [a]\n tool: test.echo\n")809 let refused = await h.engine().run(cyclic, options: h.options())810 #expect(refused.status == .failed && refused.errors.contains { $0.contains("dependency cycle") })811 let needsParam = try workflow(" - id: a\n tool: test.echo\n", spec: " parameters:\n - {name: folder, type: path, required: true}\n - {name: n, type: integer, default: 1}\n")812 let missing = await h.engine().run(needsParam, options: h.options())813 #expect(missing.status == .failed && missing.errors == ["missing required parameter 'folder'"])814 let wrongType = await h.engine().run(needsParam, options: h.options(["folder": "/x", "n": "many"]))815 #expect(wrongType.errors == ["parameter 'n' must be of type integer"])816 let unknown = await h.engine().run(needsParam, options: h.options(["folder": "/x", "typo": 1]))817 #expect(unknown.errors == ["unknown parameter 'typo'"])818 #expect(await h.tools.calls.isEmpty)819 }820}821822// MARK: - Control: cancellation, debugger823824@Suite("Engine control handle (mocked tools)") struct EngineControlTests {825 @Test func cancellationReachesRunningSteps() async throws {826 let h = try Harness(); defer { h.tmp.remove() }827 let timeline = Timeline()828 await h.tools.register("test.sleep", handler: sleepingTool(timeline, seconds: 10))829 await h.registerBasics()830 let doc = try workflow("""831 - id: one832 tool: test.sleep833 with: {id: one}834 - id: two835 tool: test.sleep836 with: {id: two}837 - id: after838 needs: [one, two]839 tool: test.echo840841 """)842 let engine = h.engine()843 let run = await engine.start(doc, options: h.options())844 #expect(await waitUntil { await run.state.running.count == 2 })845 #expect(await engine.activeRunIDs == ["run-1"])846 let began = Date()847 await run.cancel()848 let outcome = await run.outcome()849 #expect(Date().timeIntervalSince(began) < 5, "cancel must not wait for the 10 s tools")850 #expect(outcome.status == .canceled)851 #expect(await timeline.canceled.sorted() == ["one", "two"], "task cancellation reached both running tools")852 #expect(outcome.step("one")?.status == .canceled && outcome.step("after") == nil)853 #expect(await h.calls("test.echo").isEmpty)854 #expect(await waitUntil { await engine.activeRunIDs.isEmpty })855 }856857 @Test func pauseStepOverAndResume() async throws {858 let h = try Harness(); defer { h.tmp.remove() }859 await h.registerBasics()860 let doc = try workflow("""861 - id: a862 tool: test.echo863 - id: b864 needs: [a]865 tool: test.echo866 - id: c867 needs: [b]868 tool: test.echo869870 """)871 let run = await h.engine().start(doc, options: h.options(startPaused: true))872 #expect(await waitUntil { let s = await run.state; return s.status == .paused && s.pausedAt == ["a"] })873 #expect(await h.tools.calls.isEmpty, "nothing runs while paused")874 await run.stepOver()875 #expect(await waitUntil { await run.state.pausedAt == ["b"] })876 #expect(await h.tools.calls.count == 1)877 #expect(await run.inspect("a")?.status == .succeeded)878 #expect(await run.state.steps["a"] == .succeeded)879 await run.stepOver()880 #expect(await waitUntil { await run.state.pausedAt == ["c"] })881 #expect(await h.tools.calls.count == 2)882 await run.resume()883 let outcome = await run.outcome()884 #expect(outcome.status == .succeeded && outcome.steps.count == 3)885 let transitions = await h.sink.events(of: .statusChanged).map(\.text)886 #expect(transitions.contains("RUNNING → PAUSED") && transitions.contains("PAUSED → RUNNING"))887 }888889 @Test func breakpointsPauseAndVariablesCanBeModified() async throws {890 let h = try Harness(); defer { h.tmp.remove() }891 await h.registerBasics()892 let doc = try workflow("""893 - id: init894 set: {greeting: hello}895 - id: say896 needs: [init]897 tool: test.echo898 with: {text: "${{ vars.greeting }} ${{ vars.who }}"}899 - id: end900 needs: [say]901 tool: test.echo902903 """, spec: " variables:\n who: world\n")904 let run = await h.engine().start(doc, options: h.options(breakpoints: ["say"]))905 var seen: [RunStatus] = []906 let watcher = Task { () -> [RunStatus] in907 var out: [RunStatus] = []908 for await s in await run.states() { if out.last != s.status { out.append(s.status) } }909 return out910 }911 #expect(await waitUntil { let s = await run.state; return s.pausedAt == ["say"] && s.status == .paused })912 #expect(await run.variables() == ["greeting": "hello", "who": "world"])913 #expect(await run.inspect("init")?.output == ["greeting": "hello"])914 await run.setVariable("who", to: "debugger")915 await run.removeBreakpoint("say")916 await run.addBreakpoint("end")917 await run.resume()918 #expect(await waitUntil { await run.state.pausedAt == ["end"] })919 #expect(await run.inspect("say")?.output == ["text": "hello debugger"])920 await run.resume()921 let outcome = await run.outcome()922 seen = await watcher.value923 #expect(outcome.status == .succeeded)924 #expect(seen.contains(.paused) && seen.last == .succeeded, "the state stream ends with the final status: \(seen)")925 #expect(await h.sink.events(of: .log).contains { $0.text.contains("breakpoint hit at 'say'") })926 }927928 @Test func pauseDuringARunHoldsTheNextStep() async throws {929 let h = try Harness(); defer { h.tmp.remove() }930 let timeline = Timeline()931 await h.tools.register("test.sleep", handler: sleepingTool(timeline, seconds: 0.2))932 await h.registerBasics()933 let doc = try workflow(" - id: slow\n tool: test.sleep\n with: {id: slow}\n - id: next\n needs: [slow]\n tool: test.echo\n")934 let run = await h.engine().start(doc, options: h.options())935 #expect(await waitUntil { await run.state.running == ["slow"] })936 await run.pause()937 #expect(await waitUntil { let s = await run.state; return s.status == .paused && s.pausedAt == ["next"] })938 #expect(await run.inspect("slow")?.status == .succeeded, "a running step finishes; the next one waits")939 #expect(await h.calls("test.echo").isEmpty)940 await run.cancel()941 #expect(await run.outcome().status == .canceled)942 }943}944945// MARK: - Checkpoints946947@Suite("Engine checkpoints and recovery (mocked tools)") struct EngineCheckpointTests {948 static let pipeline = """949 - id: a950 tool: test.echo951 with: {v: 1}952 - id: b953 needs: [a]954 tool: TOOL955 with: {v: "${{ steps.a.output.v + 1 }}"}956 - id: c957 needs: [b]958 tool: test.effect959 with: {v: "${{ steps.b.output.v + 1 }}"}960 - id: out961 needs: [c]962 output: {v: "${{ steps.c.output.v }}"}963964 """965966 /// Runs the pipeline once and returns the checkpoint saved while `b` was mid-flight (a simulated crash point).967 func crashedDuringB(_ h: Harness, tool: String) async throws -> (WorkflowDocument, WorkflowCheckpoint) {968 await h.registerBasics()969 let doc = try workflow(Self.pipeline.replacingOccurrences(of: "TOOL", with: tool), spec: " variables:\n seed: 7\n")970 let first = await h.engine().run(doc, options: h.options())971 #expect(first.status == .succeeded && first.outputs == ["v": 3])972 let history = await h.store.history973 let crash = try #require(history.first { $0.inFlight.map(\.stepID) == ["b"] && $0.results["a"]?.status == .succeeded })974 #expect(crash.results["b"] == nil)975 return (doc, crash)976 }977978 @Test func checkpointsAreSavedAfterEveryStepAndRemovedOnSuccess() async throws {979 let h = try Harness(); defer { h.tmp.remove() }980 let (_, crash) = try await crashedDuringB(h, tool: "test.echo")981 let history = await h.store.history982 #expect(history.map(\.sequence) == Array(1...history.count), "sequence increases with every save")983 #expect(history.count >= 8, "one save when a step starts and one when it finishes")984 #expect(crash.variables[""] == ["seed": 7] && crash.document.spec.steps.count == 4)985 #expect(await h.store.latest(runID: "run-1") == nil, "a finished successful run leaves no checkpoint behind")986 let data = try JSONEncoder().encode(crash)987 #expect(try JSONDecoder().decode(WorkflowCheckpoint.self, from: data) == crash, "checkpoints are Codable for durable stores")988 }989990 @Test func resumeDoesNotReRunCompletedStepsAndReRunsAnIdempotentInFlightStep() async throws {991 let h = try Harness(); defer { h.tmp.remove() }992 let (_, crash) = try await crashedDuringB(h, tool: "test.echo")993 let before = await h.tools.calls.count994 let resumed = await h.engine().resume(from: crash, options: h.options()).outcome()995 #expect(resumed.status == .succeeded && resumed.outputs == ["v": 3])996 let after = await h.tools.calls.dropFirst(before)997 #expect(after.map(\.toolID) == ["test.echo", "test.effect"], "a is not run again; b (idempotent, mid-flight) and c run")998 #expect(after.first?.arguments == ["v": 2])999 #expect(await h.sink.events(of: .statusChanged).map(\.text).contains("RECOVERING → RUNNING"))1000 #expect(await h.sink.events(of: .log).contains { $0.text.contains("1 completed step(s) will not run again") })1001 }10021003 @Test func nonIdempotentInFlightStepLeadsToRecoveryRequired() async throws {1004 let h = try Harness(); defer { h.tmp.remove() }1005 let (_, crash) = try await crashedDuringB(h, tool: "test.effect")1006 #expect(crash.inFlight == [InFlightStep(key: "b", stepID: "b", kind: "tool", idempotent: false, isContainer: false)])1007 let before = await h.tools.calls.count1008 let blocked = await h.engine().resume(from: crash, options: h.options()).outcome()1009 #expect(blocked.status == .recoveryRequired)1010 #expect(await h.tools.calls.count == before, "nothing is executed, least of all the interrupted step")1011 #expect(blocked.step("b")?.status == .recoveryRequired && blocked.errors.first?.contains("not idempotent") == true)1012 #expect(blocked.checkpoint == crash, "the interrupted checkpoint stays the one to resume from")1013 var legal = RunStatus.queued1014 for e in await h.sink.events(of: .statusChanged).suffix(1) {1015 legal = RunStatus(rawValue: e.data["from"]?.stringValue ?? "") ?? .queued1016 #expect(legal == .recovering && e.data["to"] == "RECOVERY_REQUIRED")1017 }10181019 // The operator confirms the effect happened: the run continues without repeating it.1020 let marked = await h.engine().resume(from: crash, options: h.options(resolutions: ["b": .markSucceeded(output: ["v": 2])])).outcome()1021 #expect(marked.status == .succeeded && marked.outputs == ["v": 3])1022 #expect(await h.tools.calls.dropFirst(before).map { $0.arguments["v"] } == [3], "only c ran")10231024 // Or decides a re-run is safe.1025 let count = await h.tools.calls.count1026 let rerun = await h.engine().resume(from: crash, options: h.options(resolutions: ["b": .rerun])).outcome()1027 #expect(rerun.status == .succeeded)1028 #expect(await h.tools.calls.dropFirst(count).map { $0.arguments["v"] } == [2, 3])1029 }10301031 @Test func resumeContinuesInsideContainers() async throws {1032 let h = try Harness(); defer { h.tmp.remove() }1033 await h.registerBasics()1034 let doc = try workflow("""1035 - id: each1036 foreach: [10, 20, 30]1037 steps:1038 - id: work1039 tool: test.echo1040 with: {n: "${{ item }}"}1041 - id: poll1042 needs: [each]1043 while: ${{ vars.n < 3 }}1044 maxIterations: 91045 steps:1046 - id: bump1047 set: {n: "${{ vars.n + 1 }}"}1048 - id: ping1049 needs: [bump]1050 tool: test.echo1051 with: {n: "${{ vars.n }}"}1052 - id: out1053 needs: [poll]1054 output: {n: "${{ vars.n }}", items: "${{ map(steps.each.output.results, r => r.work.n) }}"}10551056 """, spec: " variables:\n n: 0\n")1057 let first = await h.engine().run(doc, options: h.options())1058 #expect(first.status == .succeeded && first.outputs == ["n": 3, "items": [10, 20, 30]])1059 let history = await h.store.history1060 // Crash while the second foreach iteration was running.1061 let mid = try #require(history.first { $0.inFlight.map(\.key) == ["each", "each[1]/work"] })1062 var count = await h.tools.calls.count1063 let resumed = await h.engine().resume(from: mid, options: h.options()).outcome()1064 #expect(resumed.status == .succeeded && resumed.outputs == ["n": 3, "items": [10, 20, 30]])1065 #expect(await h.tools.calls.dropFirst(count).map { $0.arguments["n"] } == [20, 30, 1, 2, 3], "iteration 0 is not repeated")1066 // Crash in the middle of loop iteration 1, after its `set` step: the counter must not be bumped twice.1067 let loopMid = try #require(history.first { $0.inFlight.map(\.key) == ["poll", "poll#1/ping"] })1068 #expect(loopMid.variables[""] == ["n": 2] && loopMid.containerState["poll"] == 1)1069 count = await h.tools.calls.count1070 let again = await h.engine().resume(from: loopMid, options: h.options()).outcome()1071 #expect(again.status == .succeeded && again.outputs["n"] == 3)1072 #expect(await h.tools.calls.dropFirst(count).map { $0.arguments["n"] } == [2, 3])1073 }1074}10751076// MARK: - Verification with real files, subworkflows, misc10771078@Suite("Engine verification (integration: real files in a temp directory)") struct EngineVerificationTests {1079 @Test func verifyChecksRealStateAndGatesSuccess() async throws {1080 let h = try Harness(); defer { h.tmp.remove() }1081 let root = h.tmp.url.path1082 await h.tools.register("test.write", sideEffect: .write, tier: .reversibleWrite, idempotent: false) { call, _ in1083 let path = call.arguments["path"]?.stringValue ?? ""1084 do {1085 try FileManager.default.createDirectory(atPath: (path as NSString).deletingLastPathComponent, withIntermediateDirectories: true)1086 try (call.arguments["content"]?.stringValue ?? "").write(toFile: path, atomically: true, encoding: .utf8)1087 return .success(["path": .string(path)], summary: "wrote")1088 } catch { return .failure(.failed, ToolFailure(code: "io", message: "\(error)")) }1089 }1090 let doc = try workflow("""1091 - id: write1092 tool: test.write1093 with: {path: "${{ params.dir }}/out/report.md", content: "# Weekly report\\nAll good.\\n"}1094 - id: empty1095 tool: test.write1096 with: {path: "${{ params.dir }}/out/empty.txt", content: ""}1097 - id: check1098 needs: [write, empty]1099 verify:1100 - fileExists: ${{ params.dir }}/out/report.md1101 - fileNonEmpty: ${{ steps.write.output.path }}1102 - fileContains: ${{ params.dir }}/out/report.md1103 text: Weekly report1104 - dirNonEmpty: ${{ params.dir }}/out1105 - equals: ["${{ basename(steps.write.output.path) }}", report.md]1106 - truthy: ${{ endsWith(steps.write.output.path, ".md") }}1107 - stepSucceeded: write1108 - id: out1109 needs: [check]1110 output: {ok: true}1111 artifacts: ["${{ params.dir }}/out/report.md"]11121113 """, spec: " parameters:\n - {name: dir, type: path, required: true}\n permissions:\n filesystem:\n read: [\"${{ params.dir }}\"]\n")1114 let outcome = await h.engine().run(doc, options: h.options(["dir": .string(root)]))1115 #expect(outcome.status == .succeeded)1116 #expect(outcome.verifications.count == 7 && outcome.verifications.allSatisfy(\.passed))1117 #expect(outcome.verifications.map(\.kind) == ["file_exists", "file_nonempty", "file_contains", "dir_nonempty", "custom", "custom", "tool_succeeded"])1118 #expect(outcome.artifacts.count == 1 && outcome.artifacts[0].bytes == 26 && outcome.artifacts[0].path.hasSuffix("/out/report.md"))1119 #expect(await h.sink.events(of: .verification).count == 7)11201121 var failing = doc1122 failing.spec.steps[2].kind = .verify(VerifyStep(assertions: [1123 .fileNonEmpty(path: "${{ params.dir }}/out/empty.txt"), .fileExists(path: "${{ params.dir }}/out/missing.md"),1124 .fileContains(path: "${{ params.dir }}/out/report.md", text: "Monthly"), .fileExists(path: "/etc/hosts"),1125 .fileExists(path: "~/.ssh/id_ed25519"), .stepSucceeded("nope"),1126 ]))1127 let h2 = try Harness(); defer { h2.tmp.remove() }1128 await h2.tools.register("test.write") { call, ctx in .success(call.arguments, summary: "pretend") }1129 let bad = await h.engine().run(failing, options: h.options(["dir": .string(root)]))1130 #expect(bad.status == .failed, "SUCCEEDED requires every executed verify to pass")1131 #expect(bad.verifications.map(\.passed) == [false, false, false, false, false, false])1132 #expect(bad.verifications[0].evidence == "0 bytes")1133 #expect(bad.verifications[3].evidence.contains("outside the workflow's declared filesystem permissions"))1134 #expect(bad.verifications[4].evidence.contains("outside") || bad.verifications[4].evidence.contains("protected"))1135 #expect(bad.step("check")?.errorCode == "verification_failed" && bad.step("out") == nil)1136 }1137}11381139@Suite("Engine subworkflows and steps (mocked tools)") struct EngineCompositionTests {1140 @Test func subworkflowReceivesInputsAndReturnsOutputs() async throws {1141 let h = try Harness(); defer { h.tmp.remove() }1142 await h.registerBasics()1143 let child = try WorkflowCodec.parse("""1144 apiVersion: orix.ai/v11145 kind: Automation1146 metadata: {id: child, name: Child}1147 spec:1148 parameters:1149 - {name: text, type: string, required: true}1150 - {name: times, type: integer, default: 2}1151 variables:1152 prefix: "${{ params.text }}:"1153 steps:1154 - id: echo1155 tool: test.echo1156 with: {text: "${{ vars.prefix }}${{ params.times }}"}1157 - id: result1158 needs: [echo]1159 output: {shout: "${{ upper(steps.echo.output.text) }}"}1160 """)1161 let parent = try workflow("""1162 - id: echo1163 tool: test.echo1164 with: {text: parent}1165 - id: call1166 needs: [echo]1167 subworkflow: child1168 inputs: {text: "${{ steps.echo.output.text }}"}1169 - id: out1170 needs: [call]1171 output: {fromChild: "${{ steps.call.output.shout }}", mine: "${{ steps.echo.output.text }}"}11721173 """)1174 let outcome = await h.engine(loader: DictionarySubworkflowLoader([child])).run(parent, options: h.options())1175 #expect(outcome.status == .succeeded && outcome.outputs == ["fromChild": "PARENT:2", "mine": "parent"])1176 #expect(outcome.results(for: "echo").map(\.key) == ["echo", "call>/echo"], "same step ids live in separate namespaces")11771178 let h2 = try Harness(); defer { h2.tmp.remove() }1179 let missing = await h2.engine(loader: DictionarySubworkflowLoader([])).run(parent, options: h2.options())1180 #expect(missing.status == .failed)1181 await h2.registerBasics()1182 let h3 = try Harness(); defer { h3.tmp.remove() }1183 await h3.registerBasics()1184 let noLoader = await h3.engine().run(parent, options: h3.options())1185 #expect(noLoader.step("call")?.error?.contains("no subworkflow loader") == true)1186 }11871188 @Test func recursionIsStoppedByTheDepthLimit() async throws {1189 let h = try Harness(); defer { h.tmp.remove() }1190 let recursive = try WorkflowCodec.parse("""1191 apiVersion: orix.ai/v11192 kind: Automation1193 metadata: {id: narcissus, name: Narcissus}1194 spec:1195 steps:1196 - id: again1197 subworkflow: narcissus1198 """)1199 let outcome = await h.engine(loader: DictionarySubworkflowLoader([recursive]), configuration: .init(maxSubworkflowDepth: 4))1200 .run(recursive, options: h.options())1201 #expect(outcome.status == .failed)1202 #expect(outcome.results(for: "again").count == 5, "four nested calls, the fifth is refused")1203 #expect(outcome.errors.joined().contains("nested deeper than 4 levels"))1204 }12051206 @Test func notifySetTransformAndSecretPlaceholders() async throws {1207 let h = try Harness(); defer { h.tmp.remove() }1208 await h.registerBasics()1209 let doc = try workflow("""1210 - id: compute1211 transform: ${{ sum(map(params.items, x => x.size)) }}1212 into: total1213 - id: login1214 needs: [compute]1215 tool: test.echo1216 with: {password: "${{ secret(\\"smtp\\") }}", total: "${{ vars.total }}"}1217 - id: tell1218 needs: [login]1219 notify: Total is ${{ vars.total }}1220 title: Sizes12211222 """, spec: " parameters:\n - name: items\n type: array\n default: [{size: 2}, {size: 40}]\n")1223 let outcome = await h.engine().run(doc, options: h.options())1224 #expect(outcome.status == .succeeded)1225 #expect(outcome.step("compute")?.output == 42)1226 #expect(outcome.step("login")?.output == ["password": ["$secret": "smtp"], "total": 42], "a secret is a reference placeholder, never a value")1227 #expect(await h.calls("notifications.post").first?.arguments == ["title": "Sizes", "body": "Total is 42"])1228 let call = try #require(await h.calls("test.echo").first)1229 #expect(call.idempotencyKey == "run-1/login" && !call.dryRun)1230 }1231}1232