SPB Git

Repository Public License

EngineTests.swift 61.2 KB 1,232 lines swift

1// ORIX OS Automation2// File: Packages/ORIXWorkflowEngine/Tests/ORIXWorkflowEngineTests/EngineTests.swift3// Description: Workflow engine tests with a closure-backed tool invoker and a fake agent runner (mocked tools), plus real-file verification in temp directories.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Testing9import ORIXCore10import ORIXTestingSupport11@testable import ORIXWorkflowEngine1213// MARK: - Test doubles1415/// Wraps the shared closure invoker and marks chosen tools as supporting dry runs (the shared double cannot).16struct DryRunAwareInvoker: ToolInvoking {17    let base: ClosureToolInvoker18    let dryRunTools: Set<String>19    func manifests() async -> [ToolManifest] {20        await base.manifests().map { m in var m = m; m.supportsDryRun = dryRunTools.contains(m.identifier); return m }21    }22    func invoke(_ call: ToolCall, context: InvocationContext) async -> ToolResult { await base.invoke(call, context: context) }23}2425actor FakeAgentRunner: AgentRunning {26    private(set) var tasks: [AgentTask] = []27    private(set) var profiles: [AgentProfile] = []28    private let reply: @Sendable (AgentTask) -> AgentOutcome29    init(reply: @escaping @Sendable (AgentTask) -> AgentOutcome) { self.reply = reply }30    func run(task: AgentTask, profile: AgentProfile, runID: String, workflowID: String?, stepID: String?) async -> AgentOutcome {31        tasks.append(task); profiles.append(profile)32        return reply(task)33    }34}3536/// Records when each call ran, to prove overlap (or its absence) with timestamps.37actor Timeline {38    private(set) var spans: [String: (start: Date, end: Date)] = [:]39    private var active = 040    private(set) var maxActive = 041    private(set) var canceled: [String] = []42    func enter() -> Date { active += 1; maxActive = max(maxActive, active); return Date() }43    func leave(_ id: String, _ start: Date) { active -= 1; spans[id] = (start, Date()) }44    func markCanceled(_ id: String) { canceled.append(id) }45    func overlap(_ a: String, _ b: String) -> Bool {46        guard let x = spans[a], let y = spans[b] else { return false }47        return x.start < y.end && y.start < x.end48    }49}5051actor Counter {52    private(set) var value = 053    func next() -> Int { value += 1; return value }54}5556struct Harness {57    let tmp: TempDirectory58    let tools = ClosureToolInvoker()59    let sink = CollectingEventSink()60    let store = InMemoryCheckpointStore(keepHistory: true)6162    init() throws { tmp = try TempDirectory(prefix: "orix-wf") }6364    func engine(invoker: (any ToolInvoking)? = nil, agents: (any AgentRunning)? = nil, loader: (any SubworkflowLoading)? = nil,65                approval: WorkflowApprovalHandler? = nil, profiles: AgentProfileResolver? = nil, clock: any WorkflowClock = SystemWorkflowClock(),66                configuration: WorkflowEngineConfiguration = .init()) -> WorkflowEngine {67        WorkflowEngine(tools: invoker ?? tools, agents: agents, events: sink, checkpoints: store, loader: loader, approval: approval,68                       agentProfiles: profiles, clock: clock, configuration: configuration)69    }7071    func options(_ parameters: [String: JSONValue] = [:], dryRun: Bool = false, breakpoints: Set<String> = [], startPaused: Bool = false,72                 resolutions: [String: InFlightResolution] = [:]) -> WorkflowRunOptions {73        WorkflowRunOptions(runID: "run-1", parameters: parameters, dryRun: dryRun, workingDirectory: tmp.url,74                           artifactsDirectory: tmp.url.appendingPathComponent("artifacts"), breakpoints: breakpoints, startPaused: startPaused,75                           inFlightResolutions: resolutions)76    }7778    /// Registers `test.echo` (returns its arguments), `test.fail` and `test.effect` (not idempotent).79    func registerBasics() async {80        await tools.register("test.echo") { call, _ in .success(call.arguments, summary: "echo") }81        await tools.register("test.fail") { _, _ in .failure(.failed, ToolFailure(code: "io", message: "disk on fire")) }82        await tools.register("test.effect", sideEffect: .write, tier: .reversibleWrite, idempotent: false) { call, _ in .success(call.arguments, summary: "effect") }83        await tools.register("notifications.post", sideEffect: .write, tier: .reversibleWrite, idempotent: false) { call, _ in .success(call.arguments, summary: "posted") }84    }8586    func calls(_ toolID: String) async -> [ToolCall] { await tools.calls.filter { $0.toolID == toolID } }87}8889func workflow(_ steps: String, spec: String = "") throws -> WorkflowDocument {90    try WorkflowCodec.parse("apiVersion: orix.ai/v1\nkind: Automation\nmetadata:\n  id: t\n  name: Test\nspec:\n" + spec + "  steps:\n" + steps)91}9293func waitUntil(timeout: Double = 5, _ condition: @Sendable () async -> Bool) async -> Bool {94    let deadline = Date().addingTimeInterval(timeout)95    while Date() < deadline {96        if await condition() { return true }97        try? await Task.sleep(nanoseconds: 2_000_000)98    }99    return await condition()100}101102func sleepingTool(_ timeline: Timeline, seconds: Double) -> ClosureToolInvoker.Handler {103    { call, _ in104        let id = call.arguments["id"]?.stringValue ?? call.id105        let start = await timeline.enter()106        do { try await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000)) }107        catch {108            await timeline.leave(id, start)109            await timeline.markCanceled(id)110            return .failure(.canceled, ToolFailure(code: "canceled", message: "Canceled"))111        }112        await timeline.leave(id, start)113        return .success(["id": .string(id)], summary: "slept")114    }115}116117// MARK: - Scheduling118119@Suite("Engine scheduling (mocked tools)") struct EngineSchedulingTests {120    @Test func stepsRunInDependencyOrderAndPassOutputs() async throws {121        let h = try Harness(); defer { h.tmp.remove() }122        await h.registerBasics()123        let doc = try workflow("""124            - id: c125              needs: [b]126              tool: test.echo127              with: {text: "${{ steps.b.output.text }}-c", status: "${{ steps.a.status }}"}128            - id: a129              tool: test.echo130              with: {text: a}131            - id: b132              needs: [a]133              tool: test.echo134              with: {text: "${{ steps.a.output.text }}-b"}135            - id: out136              needs: [c]137              output: {final: "${{ steps.c.output.text }}", count: "${{ len(steps.c.output.text) }}"}138139        """)140        let outcome = await h.engine().run(doc, options: h.options())141        #expect(outcome.status == .succeeded)142        #expect(outcome.steps.map(\.stepID) == ["a", "b", "c", "out"])143        #expect(outcome.outputs == ["final": "a-b-c", "count": 5])144        #expect(outcome.step("c")?.output["status"] == "succeeded")145        #expect(outcome.usage.toolCalls == 3)146        let started = await h.sink.events(of: .stepStarted).map(\.stepID)147        #expect(started == ["a", "b", "c", "out"])148        let finished = await h.sink.events(of: .stepFinished)149        #expect(finished.count == 4 && finished.allSatisfy { $0.data["status"] == "succeeded" })150        #expect(await h.sink.events(of: .checkpoint).count == 4)151        #expect(outcome.steps.allSatisfy { $0.attempts == 1 && $0.startedAt != nil && $0.endedAt != nil })152    }153154    @Test func independentStepsOverlapInTime() async throws {155        let h = try Harness(); defer { h.tmp.remove() }156        let timeline = Timeline()157        await h.tools.register("test.sleep", handler: sleepingTool(timeline, seconds: 0.25))158        let doc = try workflow("""159            - id: left160              tool: test.sleep161              with: {id: left}162            - id: right163              tool: test.sleep164              with: {id: right}165            - id: join166              needs: [left, right]167              tool: test.sleep168              with: {id: join}169170        """)171        let began = Date()172        let outcome = await h.engine().run(doc, options: h.options())173        let elapsed = Date().timeIntervalSince(began)174        #expect(outcome.status == .succeeded)175        #expect(await timeline.overlap("left", "right"), "independent steps must run at the same time")176        let leftJoin = await timeline.overlap("left", "join"), rightJoin = await timeline.overlap("right", "join")177        #expect(!leftJoin && !rightJoin)178        #expect(await timeline.maxActive == 2)179        #expect(elapsed < 0.74, "three 0.25 s steps took \(elapsed) s; two of them must have overlapped")180    }181182    @Test func maxConcurrentStepsBoundsParallelism() async throws {183        let h = try Harness(); defer { h.tmp.remove() }184        let timeline = Timeline()185        await h.tools.register("test.sleep", handler: sleepingTool(timeline, seconds: 0.05))186        let doc = try workflow((1...5).map { "    - id: s\($0)\n      tool: test.sleep\n      with: {id: s\($0)}\n" }.joined())187        let outcome = await h.engine(configuration: .init(maxConcurrentSteps: 2)).run(doc, options: h.options())188        #expect(outcome.status == .succeeded)189        #expect(await timeline.maxActive == 2)190        let serial = Timeline()191        let h2 = try Harness(); defer { h2.tmp.remove() }192        await h2.tools.register("test.sleep", handler: sleepingTool(serial, seconds: 0.03))193        _ = await h2.engine(configuration: .init(maxConcurrentSteps: 1)).run(doc, options: h2.options())194        #expect(await serial.maxActive == 1)195    }196197    @Test func conditionRoutesBranchesAndSkipsPropagate() async throws {198        let h = try Harness(); defer { h.tmp.remove() }199        await h.registerBasics()200        let doc = try workflow("""201            - id: check202              if: ${{ params.n > 2 }}203              then: [big]204              else: [small]205            - id: big206              tool: test.echo207              with: {size: big}208            - id: small209              tool: test.echo210              with: {size: small}211            - id: afterSmall212              needs: [small]213              tool: test.echo214            - id: gated215              when: ${{ params.n == 0 }}216              tool: test.echo217            - id: always218              needs: [check]219              tool: test.echo220221        """, spec: "  parameters:\n    - {name: n, type: integer, default: 1}\n")222        let big = await h.engine().run(doc, options: h.options(["n": 5]))223        #expect(big.status == .succeeded)224        #expect(big.step("check")?.output == ["result": true, "branch": "then"])225        #expect(big.step("big")?.status == .succeeded)226        #expect(big.step("small")?.status == .skipped && big.step("small")?.note?.contains("'then' branch") == true)227        #expect(big.step("afterSmall")?.status == .skipped && big.step("afterSmall")?.note?.contains("'small' was skipped") == true)228        #expect(big.step("gated")?.status == .skipped && big.step("gated")?.note?.contains("when") == true)229        #expect(big.step("always")?.status == .succeeded)230        #expect(await h.calls("test.echo").count == 2)231232        let h2 = try Harness(); defer { h2.tmp.remove() }233        await h2.registerBasics()234        let small = await h2.engine().run(doc, options: h2.options())235        #expect(small.step("big")?.status == .skipped && small.step("small")?.status == .succeeded && small.step("afterSmall")?.status == .succeeded)236    }237238    @Test func switchSelectsCaseOrDefault() async throws {239        let doc = try workflow("""240            - id: pick241              switch: ${{ params.mode }}242              cases:243                - value: fast244                  steps: [quick]245                - value: 2246                  steps: [two]247              default: [fallback]248            - id: quick249              tool: test.echo250            - id: two251              tool: test.echo252            - id: fallback253              tool: test.echo254255        """, spec: "  parameters:\n    - {name: mode, type: string, default: fast}\n")256        for (mode, expected) in [("fast", "quick"), ("other", "fallback")] {257            let h = try Harness(); defer { h.tmp.remove() }258            await h.registerBasics()259            let outcome = await h.engine().run(doc, options: h.options(["mode": .string(mode)]))260            #expect(outcome.status == .succeeded)261            for id in ["quick", "two", "fallback"] { #expect(outcome.step(id)?.status == (id == expected ? .succeeded : .skipped), "\(mode): \(id)") }262        }263    }264265    @Test func foreachFansOutInOrderWithBoundedConcurrency() async throws {266        let h = try Harness(); defer { h.tmp.remove() }267        let timeline = Timeline()268        await h.tools.register("test.sleep", handler: sleepingTool(timeline, seconds: 0.03))269        await h.registerBasics()270        let doc = try workflow("""271            - id: each272              foreach: ${{ params.names }}273              as: name274              concurrency: 3275              collect: shout276              steps:277                - id: nap278                  tool: test.sleep279                  with: {id: "${{ name }}"}280                - id: shout281                  needs: [nap]282                  transform: ${{ upper(name) + string(index) }}283            - id: all284              needs: [each]285              output: {names: "${{ steps.each.output.results }}", count: "${{ steps.each.output.count }}"}286287        """, spec: "  parameters:\n    - {name: names, type: array, default: [a, b, c, d, e, f, g]}\n")288        let outcome = await h.engine(configuration: .init(maxConcurrentSteps: 8)).run(doc, options: h.options())289        #expect(outcome.status == .succeeded)290        #expect(outcome.outputs["names"] == ["A0", "B1", "C2", "D3", "E4", "F5", "G6"])291        #expect(outcome.outputs["count"] == 7)292        #expect(await timeline.maxActive == 3, "foreach concurrency bounds the fan-out")293        #expect(outcome.results(for: "shout").map(\.key).sorted().first == "each[0]/shout")294        #expect(outcome.results(for: "nap").count == 7)295    }296297    @Test func parallelBranchesRunTogetherAndJoin() async throws {298        let h = try Harness(); defer { h.tmp.remove() }299        let timeline = Timeline()300        await h.tools.register("test.sleep", handler: sleepingTool(timeline, seconds: 0.15))301        let doc = try workflow("""302            - id: both303              parallel:304                - name: left305                  steps:306                    - id: l307                      tool: test.sleep308                      with: {id: l}309                - name: right310                  steps:311                    - id: r312                      tool: test.sleep313                      with: {id: r}314            - id: after315              needs: [both]316              output: {left: "${{ steps.both.output.left.l.id }}", right: "${{ steps.both.output.right.r.id }}"}317318        """)319        let outcome = await h.engine().run(doc, options: h.options())320        #expect(outcome.status == .succeeded && outcome.outputs == ["left": "l", "right": "r"])321        #expect(await timeline.overlap("l", "r"))322    }323324    @Test func loopRunsUntilConditionIsFalseAndStopsAtItsBound() async throws {325        let h = try Harness(); defer { h.tmp.remove() }326        await h.registerBasics()327        let doc = try workflow("""328            - id: poll329              while: ${{ vars.n < params.target }}330              maxIterations: 5331              steps:332                - id: bump333                  set: {n: "${{ vars.n + 1 }}", seen: "${{ loop.index }}"}334            - id: done335              needs: [poll]336              output: {n: "${{ vars.n }}", iterations: "${{ steps.poll.output.iterations }}"}337338        """, spec: "  parameters:\n    - {name: target, type: integer, default: 3}\n  variables:\n    n: 0\n")339        let ok = await h.engine().run(doc, options: h.options())340        #expect(ok.status == .succeeded && ok.outputs == ["n": 3, "iterations": 3])341        #expect(ok.results(for: "bump").map(\.key) == ["poll#0/bump", "poll#1/bump", "poll#2/bump"])342343        let h2 = try Harness(); defer { h2.tmp.remove() }344        let bounded = await h2.engine().run(doc, options: h2.options(["target": 99]))345        #expect(bounded.status == .failed)346        #expect(bounded.step("poll")?.errorCode == "loop_bound" && bounded.results(for: "bump").count == 5)347        #expect(bounded.step("done") == nil, "nothing runs after an unhandled failure")348    }349350    @Test func foreachRefusesNonLists() async throws {351        let h = try Harness(); defer { h.tmp.remove() }352        let doc = try workflow("    - id: each\n      foreach: ${{ 42 }}\n      steps:\n        - id: x\n          delay: 0\n")353        let outcome = await h.engine().run(doc, options: h.options())354        #expect(outcome.status == .failed && outcome.step("each")?.error?.contains("needs a list") == true)355    }356}357358// MARK: - Failure handling359360@Suite("Engine retries, timeouts and error routing (mocked tools)") struct EngineFailureTests {361    static let flakyDoc = """362        - id: flaky363          tool: TOOL364          retry: {maxAttempts: 4, backoffSeconds: 1, backoffMultiplier: 2FORCE}365366    """367368    func runFlaky(tool: String, idempotent: Bool, force: Bool) async throws -> (WorkflowOutcome, [Double], [RunEvent], Int) {369        let h = try Harness(); defer { h.tmp.remove() }370        let counter = Counter()371        await h.tools.register(tool, sideEffect: idempotent ? .read : .write, tier: idempotent ? .readOnly : .reversibleWrite, idempotent: idempotent) { _, _ in372            let n = await counter.next()373            return n < 3 ? .failure(.failed, ToolFailure(code: "network", message: "attempt \(n) failed", retryable: true)) : .success(["attempt": .int(n)], summary: "ok")374        }375        let clock = VirtualWorkflowClock()376        let doc = try workflow(Self.flakyDoc.replacingOccurrences(of: "TOOL", with: tool).replacingOccurrences(of: "FORCE", with: force ? ", force: true" : ""))377        let outcome = await h.engine(clock: clock).run(doc, options: h.options())378        return (outcome, clock.sleeps, await h.sink.events(of: .retry), await counter.value)379    }380381    @Test func idempotentToolIsRetriedWithBackoff() async throws {382        let (outcome, sleeps, events, calls) = try await runFlaky(tool: "test.flakyRead", idempotent: true, force: false)383        #expect(outcome.status == .succeeded)384        #expect(outcome.step("flaky")?.attempts == 3 && calls == 3)385        #expect(sleeps == [1, 2], "exponential backoff through the injected clock")386        #expect(events.count == 2 && events.allSatisfy { $0.data["retried"] == true && $0.stepID == "flaky" })387    }388389    @Test func nonIdempotentToolIsNotBlindlyRetriedAndTheEventSaysWhy() async throws {390        let (outcome, sleeps, events, calls) = try await runFlaky(tool: "test.flakyWrite", idempotent: false, force: false)391        #expect(outcome.status == .failed)392        #expect(outcome.step("flaky")?.attempts == 1 && calls == 1 && sleeps.isEmpty)393        #expect(events.count == 1 && events[0].data["retried"] == false)394        #expect(events[0].text.contains("not idempotent") && events[0].text.contains("retry.force"))395    }396397    @Test func forceRetriesANonIdempotentTool() async throws {398        let (outcome, _, events, calls) = try await runFlaky(tool: "test.flakyWrite", idempotent: false, force: true)399        #expect(outcome.status == .succeeded && calls == 3)400        #expect(events.count == 2 && events.allSatisfy { $0.data["forced"] == true })401    }402403    @Test func retriesStopAtMaxAttempts() async throws {404        let h = try Harness(); defer { h.tmp.remove() }405        await h.registerBasics()406        let doc = try workflow("    - id: f\n      tool: test.fail\n      retry: {maxAttempts: 3}\n")407        let outcome = await h.engine().run(doc, options: h.options())408        #expect(outcome.status == .failed && outcome.step("f")?.attempts == 3)409        #expect(await h.calls("test.fail").count == 3)410        #expect(outcome.errors.contains { $0.contains("disk on fire") })411    }412413    @Test func stepTimeoutCancelsTheToolAndMapsToTimedOut() async throws {414        let h = try Harness(); defer { h.tmp.remove() }415        let timeline = Timeline()416        await h.tools.register("test.sleep", handler: sleepingTool(timeline, seconds: 10))417        await h.registerBasics()418        let doc = try workflow("""419            - id: slow420              tool: test.sleep421              with: {id: slow}422              timeoutSeconds: 0.1423            - id: after424              needs: [slow]425              tool: test.echo426427        """)428        let began = Date()429        let outcome = await h.engine().run(doc, options: h.options())430        #expect(Date().timeIntervalSince(began) < 5)431        #expect(outcome.status == .timedOut)432        #expect(outcome.step("slow")?.status == .timedOut && outcome.step("slow")?.errorCode == "timeout")433        #expect(await timeline.canceled == ["slow"], "the timeout must cancel the running tool")434        #expect(outcome.step("after") == nil)435436        let h2 = try Harness(); defer { h2.tmp.remove() }437        await h2.tools.register("test.sleep", handler: sleepingTool(Timeline(), seconds: 10))438        await h2.registerBasics()439        var tolerant = doc440        tolerant.spec.steps[0].continueOnError = true441        let partial = await h2.engine().run(tolerant, options: h2.options())442        #expect(partial.status == .partiallySucceeded && partial.step("after")?.status == .succeeded)443    }444445    @Test func continueOnErrorYieldsPartialSuccess() async throws {446        let h = try Harness(); defer { h.tmp.remove() }447        await h.registerBasics()448        let doc = try workflow("""449            - id: bad450              tool: test.fail451              continueOnError: true452            - id: next453              needs: [bad]454              tool: test.echo455              with: {saw: "${{ steps.bad.status }}", why: "${{ steps.bad.error }}"}456457        """)458        let outcome = await h.engine().run(doc, options: h.options())459        #expect(outcome.status == .partiallySucceeded)460        #expect(outcome.step("bad")?.status == .failed && outcome.step("bad")?.continued == true)461        #expect(outcome.step("next")?.output == ["saw": "failed", "why": "disk on fire"])462    }463464    @Test func onErrorRoutesToACompensationStep() async throws {465        let h = try Harness(); defer { h.tmp.remove() }466        await h.registerBasics()467        let doc = try workflow("""468            - id: write469              tool: test.fail470              onError: undo471            - id: undo472              tool: test.echo473              with: {compensating: "${{ error.stepID }}", because: "${{ error.message }}", status: "${{ steps.write.status }}"}474            - id: next475              needs: [write]476              tool: test.echo477              with: {handled: yes}478479        """)480        let outcome = await h.engine().run(doc, options: h.options())481        #expect(outcome.status == .partiallySucceeded)482        #expect(outcome.step("write")?.handledBy == "undo" && outcome.step("write")?.continued == true)483        #expect(outcome.step("undo")?.output == ["compensating": "write", "because": "disk on fire", "status": "failed"])484        #expect(outcome.step("next")?.status == .succeeded)485        #expect(outcome.steps.map(\.stepID) == ["undo", "write", "next"])486    }487488    @Test func handlerOnlyRunsOnFailureAndAFailingHandlerFailsTheRun() async throws {489        let h = try Harness(); defer { h.tmp.remove() }490        await h.registerBasics()491        let fine = try workflow("    - id: ok\n      tool: test.echo\n      onError: undo\n    - id: undo\n      tool: test.echo\n")492        let clean = await h.engine().run(fine, options: h.options())493        #expect(clean.status == .succeeded && clean.step("undo") == nil, "a handler step never runs in the normal flow")494495        let h2 = try Harness(); defer { h2.tmp.remove() }496        await h2.registerBasics()497        let broken = try workflow("    - id: bad\n      tool: test.fail\n      onError: undo\n    - id: undo\n      fail: cannot undo\n")498        let failed = await h2.engine().run(broken, options: h2.options())499        #expect(failed.status == .failed && failed.step("bad")?.handledBy == nil && failed.step("undo")?.status == .failed)500    }501502    @Test func workflowLevelOnErrorRunsAfterAFailure() async throws {503        let h = try Harness(); defer { h.tmp.remove() }504        await h.registerBasics()505        let doc = try workflow("    - id: bad\n      fail: \"stop: ${{ 1 + 1 }}\"\n",506                               spec: "  onError:\n    - id: alert\n      notify: \"${{ error.stepID }} failed: ${{ error.message }}\"\n")507        let outcome = await h.engine().run(doc, options: h.options())508        #expect(outcome.status == .failed)509        #expect(outcome.step("bad")?.error == "stop: 2" && outcome.step("alert")?.status == .succeeded)510        #expect(await h.calls("notifications.post").first?.arguments == ["title": "Test", "body": "bad failed: stop: 2"])511    }512513    @Test func failureInsideAContainerCanBeHandledByTheContainer() async throws {514        let h = try Harness(); defer { h.tmp.remove() }515        await h.registerBasics()516        let doc = try workflow("""517            - id: each518              foreach: [1, 2, 3]519              continueOnError: true520              steps:521                - id: maybe522                  when: ${{ item == 2 }}523                  tool: test.fail524            - id: after525              needs: [each]526              tool: test.echo527528        """)529        let outcome = await h.engine().run(doc, options: h.options())530        #expect(outcome.status == .partiallySucceeded)531        #expect(outcome.step("each")?.status == .failed && outcome.step("each")?.error?.contains("iteration 1") == true)532        #expect(outcome.step("after")?.status == .succeeded)533    }534535    @Test func expressionErrorsFailTheStepWithADiagnostic() async throws {536        let h = try Harness(); defer { h.tmp.remove() }537        await h.registerBasics()538        let doc = try workflow("    - id: bad\n      tool: test.echo\n      with: {x: \"${{ 1 / 0 }}\"}\n")539        let outcome = await h.engine().run(doc, options: h.options())540        #expect(outcome.status == .failed && outcome.step("bad")?.errorCode == "expression")541        #expect(outcome.diagnostics.first?.stepID == "bad" && outcome.diagnostics.first?.message.contains("division by zero") == true)542        #expect(await h.tools.calls.isEmpty)543    }544545    @Test func interpolatedToolOutputIsNeverReEvaluated() async throws {546        let h = try Harness(); defer { h.tmp.remove() }547        await h.registerBasics()548        await h.tools.register("test.hostile") { _, _ in .success(["text": "${{ secret(\"api\") }} ${{ env.HOME }}"], summary: "web page") }549        let doc = try workflow("""550            - id: fetch551              tool: test.hostile552            - id: use553              needs: [fetch]554              tool: test.echo555              with: {exact: "${{ steps.fetch.output.text }}", mixed: "got ${{ steps.fetch.output.text }}"}556557        """)558        let outcome = await h.engine().run(doc, options: h.options())559        #expect(outcome.status == .succeeded)560        #expect(outcome.step("use")?.output == ["exact": "${{ secret(\"api\") }} ${{ env.HOME }}", "mixed": "got ${{ secret(\"api\") }} ${{ env.HOME }}"])561    }562}563564// MARK: - Approvals, dry run, budgets, status mapping565566@Suite("Engine approvals, dry run, budgets and statuses (mocked tools)") struct EnginePolicyTests {567    static let gated = """568        - id: prepare569          tool: test.effect570          with: {n: 1}571        - id: gate572          needs: [prepare]573          approval: "Send ${{ steps.prepare.output.n }} file?"574        - id: send575          needs: [gate]576          tool: test.effect577          with: {n: 2}578579    """580581    @Test func approvalApprovedDeniedAndUnanswered() async throws {582        let h = try Harness(); defer { h.tmp.remove() }583        await h.registerBasics()584        let doc = try workflow(Self.gated)585        let approved = await h.engine(approval: { prompt in586            #expect(prompt.message == "Send 1 file?" && prompt.stepID == "gate")587            return .approveOnce588        }).run(doc, options: h.options())589        #expect(approved.status == .succeeded && approved.step("gate")?.output == ["approved": true])590        #expect(await h.sink.events(of: .approvalRequested).count == 1)591        #expect(await h.sink.events(of: .approvalResolved).count == 1)592        let transitions = await h.sink.events(of: .statusChanged).map(\.text)593        #expect(transitions.contains("RUNNING → WAITING_APPROVAL") && transitions.contains("WAITING_APPROVAL → RUNNING"))594595        let h2 = try Harness(); defer { h2.tmp.remove() }596        await h2.registerBasics()597        let denied = await h2.engine(approval: { _ in .deny(reason: "not today") }).run(doc, options: h2.options())598        #expect(denied.status == .failed && denied.step("gate")?.error == "approval denied: not today")599        #expect(await h2.calls("test.effect").count == 1)600601        let h3 = try Harness(); defer { h3.tmp.remove() }602        await h3.registerBasics()603        let unanswered = await h3.engine().run(doc, options: h3.options())604        #expect(unanswered.status == .waitingApproval && unanswered.step("gate")?.status == .notApproved)605        #expect(await h3.calls("test.effect").count == 1, "nothing after the gate may run")606607        // The run can be resumed from its checkpoint once somebody answers: the completed step is not run again.608        let checkpoint = try #require(unanswered.checkpoint)609        let resumed = await h3.engine(approval: { _ in .approveForRun }).resume(from: checkpoint, options: h3.options()).outcome()610        #expect(resumed.status == .succeeded)611        #expect(await h3.calls("test.effect").map { $0.arguments["n"] } == [1, 2])612    }613614    @Test func dryRunExecutesNothingAndSaysExactlyWhatWasSimulated() async throws {615        let h = try Harness(); defer { h.tmp.remove() }616        await h.registerBasics()617        await h.tools.register("test.plan", sideEffect: .write, tier: .reversibleWrite, idempotent: false) { call, _ in618            call.dryRun ? .simulated(["wouldMove": 3], summary: "would move 3 files") : .success(["moved": 3], summary: "moved 3 files")619        }620        let agent = FakeAgentRunner { _ in AgentOutcome(status: .succeeded) }621        let invoker = DryRunAwareInvoker(base: h.tools, dryRunTools: ["test.plan"])622        let doc = try workflow("""623            - id: plan624              tool: test.plan625              with: {path: /tmp/x}626            - id: effect627              needs: [plan]628              tool: test.effect629            - id: think630              needs: [effect]631              agent: {instructions: Think, model: anthropic/test}632              objective: Decide633            - id: gate634              needs: [think]635              approval: Go?636            - id: wait637              needs: [gate]638              delay: 3600639            - id: check640              needs: [wait]641              verify:642                - truthy: ${{ steps.effect.output.done }}643            - id: tell644              needs: [check]645              notify: Finished646            - id: out647              needs: [tell]648              output: {planned: "${{ steps.plan.output.wouldMove }}"}649650        """)651        let began = Date()652        let outcome = await h.engine(invoker: invoker, agents: agent, approval: { _ in Issue.record("no approval in a dry run"); return .approveOnce })653            .run(doc, options: h.options(dryRun: true))654        #expect(Date().timeIntervalSince(began) < 5)655        #expect(outcome.status == .succeeded && outcome.dryRun)656        let calls = await h.tools.calls657        #expect(calls.map(\.toolID) == ["test.plan"], "only dry-run capable tools are invoked")658        #expect(calls.allSatisfy { $0.dryRun }, "every invocation carries dryRun: true")659        #expect(await agent.tasks.isEmpty)660        #expect(outcome.steps.filter { $0.stepID != "out" }.allSatisfy { $0.status == .simulated })661        #expect(outcome.step("plan")?.note == "simulated: test.plan was invoked with dryRun: true — would move 3 files")662        #expect(outcome.step("effect")?.note == "simulated: not executed (test.effect: the tool has no dry-run mode)")663        #expect(outcome.step("think")?.note?.contains("agent was not run") == true)664        #expect(outcome.step("tell")?.note?.hasPrefix("simulated: not executed (notifications.post") == true)665        #expect(outcome.simulations.count == 7 && outcome.simulations[0].hasPrefix("plan: simulated:"))666        #expect(outcome.outputs == ["planned": 3])667        #expect(outcome.verifications.isEmpty)668    }669670    @Test func toolCallBudgetStopsTheRun() async throws {671        let h = try Harness(); defer { h.tmp.remove() }672        await h.registerBasics()673        let doc = try workflow("""674            - id: a675              tool: test.echo676            - id: b677              needs: [a]678              tool: test.echo679            - id: c680              needs: [b]681              tool: test.echo682683        """, spec: "  budget:\n    maxToolCalls: 2\n")684        let outcome = await h.engine().run(doc, options: h.options())685        #expect(outcome.status == .budgetExceeded)686        #expect(outcome.step("c")?.status == .budgetExceeded && outcome.usage.toolCalls == 2)687        #expect(await h.tools.calls.count == 2)688        #expect(outcome.errors.contains { $0.contains("maxToolCalls") })689    }690691    @Test func agentCostBudgetStopsTheRunAndUsageIsAggregated() async throws {692        let h = try Harness(); defer { h.tmp.remove() }693        await h.registerBasics()694        let agent = FakeAgentRunner { task in695            var usage = RunUsage()696            usage.turns = 2; usage.toolCalls = 1; usage.costUSD = 0.75697            usage.tokens = TokenUsage(inputTokens: 100, outputTokens: 50)698            return AgentOutcome(status: .succeeded, finalText: "done: \(task.objective)", result: ["n": 1], usage: usage)699        }700        let doc = try workflow("""701            - id: first702              agent: writer703              objective: Draft ${{ params.topic }}704              inputs: {topic: "${{ params.topic }}"}705              expect:706                - fileNonEmpty: ~/out/${{ params.topic }}.md707            - id: second708              needs: [first]709              agent: writer710              objective: Refine ${{ steps.first.output.text }}711            - id: third712              needs: [second]713              tool: test.echo714715        """, spec: "  parameters:\n    - {name: topic, type: string, default: cats}\n  budget:\n    maxProviderCostUSD: 1\n")716        let profile = AgentProfile(name: "writer", instructions: "Write.", model: ModelRef(provider: .anthropic, model: "test"))717        let outcome = await h.engine(agents: agent, profiles: { $0 == "writer" ? profile : nil }).run(doc, options: h.options())718        #expect(outcome.status == .budgetExceeded)719        #expect(outcome.usage.costUSD == 1.5 && outcome.usage.turns == 4 && outcome.usage.tokens.totalTokens == 300 && outcome.usage.toolCalls == 2)720        #expect(outcome.step("second")?.status == .succeeded && outcome.step("third") == nil)721        let tasks = await agent.tasks722        #expect(tasks.count == 2)723        #expect(tasks[0].objective == "Draft cats" && tasks[0].inputs == ["topic": "cats"])724        #expect(tasks[0].expectations == [Expectation(kind: "file_nonempty", path: "~/out/cats.md")])725        #expect(tasks[0].budget.maxCostUSD == 1 && tasks[1].budget.maxCostUSD == 0.25, "the agent only gets what is left of the workflow budget")726        #expect(tasks[1].objective == "Refine done: Draft cats")727    }728729    @Test func agentStepNeedsARuntimeAndAModel() async throws {730        let h = try Harness(); defer { h.tmp.remove() }731        let doc = try workflow("    - id: a\n      agent: {instructions: Hi}\n      objective: Go\n")732        let noRuntime = await h.engine().run(doc, options: h.options())733        #expect(noRuntime.status == .failed && noRuntime.step("a")?.error?.contains("no agent runtime") == true)734        let agent = FakeAgentRunner { _ in AgentOutcome(status: .succeeded) }735        let h2 = try Harness(); defer { h2.tmp.remove() }736        let noModel = await h2.engine(agents: agent).run(doc, options: h2.options())737        #expect(noModel.status == .failed && noModel.step("a")?.error?.contains("no default model") == true)738        let h3 = try Harness(); defer { h3.tmp.remove() }739        let ok = await h3.engine(agents: agent, configuration: .init(defaultModel: ModelRef(provider: .openai, model: "m"))).run(doc, options: h3.options())740        #expect(ok.status == .succeeded)741        let inline = await agent.profiles.first742        #expect(inline?.model == ModelRef(provider: .openai, model: "m") && inline?.toolPatterns == [])743    }744745    @Test func durationBudgetMapsToTimedOut() async throws {746        let h = try Harness(); defer { h.tmp.remove() }747        await h.registerBasics()748        let doc = try workflow("""749            - id: wait750              delay: 120751            - id: late752              needs: [wait]753              tool: test.echo754755        """, spec: "  budget:\n    maxDurationSeconds: 60\n")756        let clock = VirtualWorkflowClock()757        let outcome = await h.engine(clock: clock).run(doc, options: h.options())758        #expect(outcome.status == .timedOut && clock.sleeps == [120])759        #expect(outcome.step("late") == nil && outcome.errors.contains { $0.contains("maxDurationSeconds") })760761        // Real time: the watchdog cancels a running step.762        let h2 = try Harness(); defer { h2.tmp.remove() }763        let timeline = Timeline()764        await h2.tools.register("test.sleep", handler: sleepingTool(timeline, seconds: 10))765        let slow = try workflow("    - id: slow\n      tool: test.sleep\n      with: {id: slow}\n", spec: "  budget:\n    maxDurationSeconds: 0.15\n")766        let real = await h2.engine().run(slow, options: h2.options())767        #expect(real.status == .timedOut && real.step("slow")?.status == .canceled)768        #expect(await timeline.canceled == ["slow"])769    }770771    @Test(arguments: [772        (ToolStatus.succeeded, RunStatus.succeeded), (.failed, .failed), (.denied, .failed), (.invalidInput, .failed), (.unsupported, .failed),773        (.notApproved, .waitingApproval), (.blockedPermission, .blockedPermission), (.timedOut, .timedOut), (.canceled, .canceled),774    ])775    func toolStatusMapsToRunStatus(tool: ToolStatus, expected: RunStatus) async throws {776        let h = try Harness(); defer { h.tmp.remove() }777        await h.registerBasics()778        await h.tools.register("test.status") { _, _ in779            tool == .succeeded ? .success(.null, summary: "ok") : .failure(tool, ToolFailure(code: tool.rawValue, message: "tool said \(tool.rawValue)"))780        }781        let doc = try workflow("    - id: s\n      tool: test.status\n    - id: after\n      needs: [s]\n      tool: test.echo\n")782        let outcome = await h.engine().run(doc, options: h.options())783        #expect(outcome.status == expected)784        #expect(outcome.step("s")?.status == WorkflowEngine.stepStatus(for: tool))785        #expect((outcome.step("after") != nil) == (tool == .succeeded))786        // Every transition the run announced is legal for the Core state machine.787        var current = RunStatus.queued788        for e in await h.sink.events(of: .statusChanged) {789            let next = try #require(RunStatus(rawValue: e.data["to"]?.stringValue ?? ""))790            #expect(current.canTransition(to: next), "\(current.rawValue) → \(next.rawValue)")791            current = next792        }793        #expect(current == expected)794    }795796    @Test func unhandledStepStatusTable() {797        let expected: [StepStatus: RunStatus] = [798            .failed: .failed, .timedOut: .timedOut, .canceled: .canceled, .notApproved: .waitingApproval,799            .blockedPermission: .blockedPermission, .budgetExceeded: .budgetExceeded, .recoveryRequired: .recoveryRequired,800        ]801        for (step, run) in expected { #expect(WorkflowEngine.runStatus(forUnhandled: step) == run) }802        #expect(StepStatus.simulated.isSuccess && StepStatus.succeeded.isSuccess && !StepStatus.skipped.isSuccess)803    }804805    @Test func invalidDocumentsAndParametersAreRefusedBeforeAnythingRuns() async throws {806        let h = try Harness(); defer { h.tmp.remove() }807        await h.registerBasics()808        let cyclic = try workflow("    - id: a\n      needs: [b]\n      tool: test.echo\n    - id: b\n      needs: [a]\n      tool: test.echo\n")809        let refused = await h.engine().run(cyclic, options: h.options())810        #expect(refused.status == .failed && refused.errors.contains { $0.contains("dependency cycle") })811        let needsParam = try workflow("    - id: a\n      tool: test.echo\n", spec: "  parameters:\n    - {name: folder, type: path, required: true}\n    - {name: n, type: integer, default: 1}\n")812        let missing = await h.engine().run(needsParam, options: h.options())813        #expect(missing.status == .failed && missing.errors == ["missing required parameter 'folder'"])814        let wrongType = await h.engine().run(needsParam, options: h.options(["folder": "/x", "n": "many"]))815        #expect(wrongType.errors == ["parameter 'n' must be of type integer"])816        let unknown = await h.engine().run(needsParam, options: h.options(["folder": "/x", "typo": 1]))817        #expect(unknown.errors == ["unknown parameter 'typo'"])818        #expect(await h.tools.calls.isEmpty)819    }820}821822// MARK: - Control: cancellation, debugger823824@Suite("Engine control handle (mocked tools)") struct EngineControlTests {825    @Test func cancellationReachesRunningSteps() async throws {826        let h = try Harness(); defer { h.tmp.remove() }827        let timeline = Timeline()828        await h.tools.register("test.sleep", handler: sleepingTool(timeline, seconds: 10))829        await h.registerBasics()830        let doc = try workflow("""831            - id: one832              tool: test.sleep833              with: {id: one}834            - id: two835              tool: test.sleep836              with: {id: two}837            - id: after838              needs: [one, two]839              tool: test.echo840841        """)842        let engine = h.engine()843        let run = await engine.start(doc, options: h.options())844        #expect(await waitUntil { await run.state.running.count == 2 })845        #expect(await engine.activeRunIDs == ["run-1"])846        let began = Date()847        await run.cancel()848        let outcome = await run.outcome()849        #expect(Date().timeIntervalSince(began) < 5, "cancel must not wait for the 10 s tools")850        #expect(outcome.status == .canceled)851        #expect(await timeline.canceled.sorted() == ["one", "two"], "task cancellation reached both running tools")852        #expect(outcome.step("one")?.status == .canceled && outcome.step("after") == nil)853        #expect(await h.calls("test.echo").isEmpty)854        #expect(await waitUntil { await engine.activeRunIDs.isEmpty })855    }856857    @Test func pauseStepOverAndResume() async throws {858        let h = try Harness(); defer { h.tmp.remove() }859        await h.registerBasics()860        let doc = try workflow("""861            - id: a862              tool: test.echo863            - id: b864              needs: [a]865              tool: test.echo866            - id: c867              needs: [b]868              tool: test.echo869870        """)871        let run = await h.engine().start(doc, options: h.options(startPaused: true))872        #expect(await waitUntil { let s = await run.state; return s.status == .paused && s.pausedAt == ["a"] })873        #expect(await h.tools.calls.isEmpty, "nothing runs while paused")874        await run.stepOver()875        #expect(await waitUntil { await run.state.pausedAt == ["b"] })876        #expect(await h.tools.calls.count == 1)877        #expect(await run.inspect("a")?.status == .succeeded)878        #expect(await run.state.steps["a"] == .succeeded)879        await run.stepOver()880        #expect(await waitUntil { await run.state.pausedAt == ["c"] })881        #expect(await h.tools.calls.count == 2)882        await run.resume()883        let outcome = await run.outcome()884        #expect(outcome.status == .succeeded && outcome.steps.count == 3)885        let transitions = await h.sink.events(of: .statusChanged).map(\.text)886        #expect(transitions.contains("RUNNING → PAUSED") && transitions.contains("PAUSED → RUNNING"))887    }888889    @Test func breakpointsPauseAndVariablesCanBeModified() async throws {890        let h = try Harness(); defer { h.tmp.remove() }891        await h.registerBasics()892        let doc = try workflow("""893            - id: init894              set: {greeting: hello}895            - id: say896              needs: [init]897              tool: test.echo898              with: {text: "${{ vars.greeting }} ${{ vars.who }}"}899            - id: end900              needs: [say]901              tool: test.echo902903        """, spec: "  variables:\n    who: world\n")904        let run = await h.engine().start(doc, options: h.options(breakpoints: ["say"]))905        var seen: [RunStatus] = []906        let watcher = Task { () -> [RunStatus] in907            var out: [RunStatus] = []908            for await s in await run.states() { if out.last != s.status { out.append(s.status) } }909            return out910        }911        #expect(await waitUntil { let s = await run.state; return s.pausedAt == ["say"] && s.status == .paused })912        #expect(await run.variables() == ["greeting": "hello", "who": "world"])913        #expect(await run.inspect("init")?.output == ["greeting": "hello"])914        await run.setVariable("who", to: "debugger")915        await run.removeBreakpoint("say")916        await run.addBreakpoint("end")917        await run.resume()918        #expect(await waitUntil { await run.state.pausedAt == ["end"] })919        #expect(await run.inspect("say")?.output == ["text": "hello debugger"])920        await run.resume()921        let outcome = await run.outcome()922        seen = await watcher.value923        #expect(outcome.status == .succeeded)924        #expect(seen.contains(.paused) && seen.last == .succeeded, "the state stream ends with the final status: \(seen)")925        #expect(await h.sink.events(of: .log).contains { $0.text.contains("breakpoint hit at 'say'") })926    }927928    @Test func pauseDuringARunHoldsTheNextStep() async throws {929        let h = try Harness(); defer { h.tmp.remove() }930        let timeline = Timeline()931        await h.tools.register("test.sleep", handler: sleepingTool(timeline, seconds: 0.2))932        await h.registerBasics()933        let doc = try workflow("    - id: slow\n      tool: test.sleep\n      with: {id: slow}\n    - id: next\n      needs: [slow]\n      tool: test.echo\n")934        let run = await h.engine().start(doc, options: h.options())935        #expect(await waitUntil { await run.state.running == ["slow"] })936        await run.pause()937        #expect(await waitUntil { let s = await run.state; return s.status == .paused && s.pausedAt == ["next"] })938        #expect(await run.inspect("slow")?.status == .succeeded, "a running step finishes; the next one waits")939        #expect(await h.calls("test.echo").isEmpty)940        await run.cancel()941        #expect(await run.outcome().status == .canceled)942    }943}944945// MARK: - Checkpoints946947@Suite("Engine checkpoints and recovery (mocked tools)") struct EngineCheckpointTests {948    static let pipeline = """949        - id: a950          tool: test.echo951          with: {v: 1}952        - id: b953          needs: [a]954          tool: TOOL955          with: {v: "${{ steps.a.output.v + 1 }}"}956        - id: c957          needs: [b]958          tool: test.effect959          with: {v: "${{ steps.b.output.v + 1 }}"}960        - id: out961          needs: [c]962          output: {v: "${{ steps.c.output.v }}"}963964    """965966    /// Runs the pipeline once and returns the checkpoint saved while `b` was mid-flight (a simulated crash point).967    func crashedDuringB(_ h: Harness, tool: String) async throws -> (WorkflowDocument, WorkflowCheckpoint) {968        await h.registerBasics()969        let doc = try workflow(Self.pipeline.replacingOccurrences(of: "TOOL", with: tool), spec: "  variables:\n    seed: 7\n")970        let first = await h.engine().run(doc, options: h.options())971        #expect(first.status == .succeeded && first.outputs == ["v": 3])972        let history = await h.store.history973        let crash = try #require(history.first { $0.inFlight.map(\.stepID) == ["b"] && $0.results["a"]?.status == .succeeded })974        #expect(crash.results["b"] == nil)975        return (doc, crash)976    }977978    @Test func checkpointsAreSavedAfterEveryStepAndRemovedOnSuccess() async throws {979        let h = try Harness(); defer { h.tmp.remove() }980        let (_, crash) = try await crashedDuringB(h, tool: "test.echo")981        let history = await h.store.history982        #expect(history.map(\.sequence) == Array(1...history.count), "sequence increases with every save")983        #expect(history.count >= 8, "one save when a step starts and one when it finishes")984        #expect(crash.variables[""] == ["seed": 7] && crash.document.spec.steps.count == 4)985        #expect(await h.store.latest(runID: "run-1") == nil, "a finished successful run leaves no checkpoint behind")986        let data = try JSONEncoder().encode(crash)987        #expect(try JSONDecoder().decode(WorkflowCheckpoint.self, from: data) == crash, "checkpoints are Codable for durable stores")988    }989990    @Test func resumeDoesNotReRunCompletedStepsAndReRunsAnIdempotentInFlightStep() async throws {991        let h = try Harness(); defer { h.tmp.remove() }992        let (_, crash) = try await crashedDuringB(h, tool: "test.echo")993        let before = await h.tools.calls.count994        let resumed = await h.engine().resume(from: crash, options: h.options()).outcome()995        #expect(resumed.status == .succeeded && resumed.outputs == ["v": 3])996        let after = await h.tools.calls.dropFirst(before)997        #expect(after.map(\.toolID) == ["test.echo", "test.effect"], "a is not run again; b (idempotent, mid-flight) and c run")998        #expect(after.first?.arguments == ["v": 2])999        #expect(await h.sink.events(of: .statusChanged).map(\.text).contains("RECOVERING → RUNNING"))1000        #expect(await h.sink.events(of: .log).contains { $0.text.contains("1 completed step(s) will not run again") })1001    }10021003    @Test func nonIdempotentInFlightStepLeadsToRecoveryRequired() async throws {1004        let h = try Harness(); defer { h.tmp.remove() }1005        let (_, crash) = try await crashedDuringB(h, tool: "test.effect")1006        #expect(crash.inFlight == [InFlightStep(key: "b", stepID: "b", kind: "tool", idempotent: false, isContainer: false)])1007        let before = await h.tools.calls.count1008        let blocked = await h.engine().resume(from: crash, options: h.options()).outcome()1009        #expect(blocked.status == .recoveryRequired)1010        #expect(await h.tools.calls.count == before, "nothing is executed, least of all the interrupted step")1011        #expect(blocked.step("b")?.status == .recoveryRequired && blocked.errors.first?.contains("not idempotent") == true)1012        #expect(blocked.checkpoint == crash, "the interrupted checkpoint stays the one to resume from")1013        var legal = RunStatus.queued1014        for e in await h.sink.events(of: .statusChanged).suffix(1) {1015            legal = RunStatus(rawValue: e.data["from"]?.stringValue ?? "") ?? .queued1016            #expect(legal == .recovering && e.data["to"] == "RECOVERY_REQUIRED")1017        }10181019        // The operator confirms the effect happened: the run continues without repeating it.1020        let marked = await h.engine().resume(from: crash, options: h.options(resolutions: ["b": .markSucceeded(output: ["v": 2])])).outcome()1021        #expect(marked.status == .succeeded && marked.outputs == ["v": 3])1022        #expect(await h.tools.calls.dropFirst(before).map { $0.arguments["v"] } == [3], "only c ran")10231024        // Or decides a re-run is safe.1025        let count = await h.tools.calls.count1026        let rerun = await h.engine().resume(from: crash, options: h.options(resolutions: ["b": .rerun])).outcome()1027        #expect(rerun.status == .succeeded)1028        #expect(await h.tools.calls.dropFirst(count).map { $0.arguments["v"] } == [2, 3])1029    }10301031    @Test func resumeContinuesInsideContainers() async throws {1032        let h = try Harness(); defer { h.tmp.remove() }1033        await h.registerBasics()1034        let doc = try workflow("""1035            - id: each1036              foreach: [10, 20, 30]1037              steps:1038                - id: work1039                  tool: test.echo1040                  with: {n: "${{ item }}"}1041            - id: poll1042              needs: [each]1043              while: ${{ vars.n < 3 }}1044              maxIterations: 91045              steps:1046                - id: bump1047                  set: {n: "${{ vars.n + 1 }}"}1048                - id: ping1049                  needs: [bump]1050                  tool: test.echo1051                  with: {n: "${{ vars.n }}"}1052            - id: out1053              needs: [poll]1054              output: {n: "${{ vars.n }}", items: "${{ map(steps.each.output.results, r => r.work.n) }}"}10551056        """, spec: "  variables:\n    n: 0\n")1057        let first = await h.engine().run(doc, options: h.options())1058        #expect(first.status == .succeeded && first.outputs == ["n": 3, "items": [10, 20, 30]])1059        let history = await h.store.history1060        // Crash while the second foreach iteration was running.1061        let mid = try #require(history.first { $0.inFlight.map(\.key) == ["each", "each[1]/work"] })1062        var count = await h.tools.calls.count1063        let resumed = await h.engine().resume(from: mid, options: h.options()).outcome()1064        #expect(resumed.status == .succeeded && resumed.outputs == ["n": 3, "items": [10, 20, 30]])1065        #expect(await h.tools.calls.dropFirst(count).map { $0.arguments["n"] } == [20, 30, 1, 2, 3], "iteration 0 is not repeated")1066        // Crash in the middle of loop iteration 1, after its `set` step: the counter must not be bumped twice.1067        let loopMid = try #require(history.first { $0.inFlight.map(\.key) == ["poll", "poll#1/ping"] })1068        #expect(loopMid.variables[""] == ["n": 2] && loopMid.containerState["poll"] == 1)1069        count = await h.tools.calls.count1070        let again = await h.engine().resume(from: loopMid, options: h.options()).outcome()1071        #expect(again.status == .succeeded && again.outputs["n"] == 3)1072        #expect(await h.tools.calls.dropFirst(count).map { $0.arguments["n"] } == [2, 3])1073    }1074}10751076// MARK: - Verification with real files, subworkflows, misc10771078@Suite("Engine verification (integration: real files in a temp directory)") struct EngineVerificationTests {1079    @Test func verifyChecksRealStateAndGatesSuccess() async throws {1080        let h = try Harness(); defer { h.tmp.remove() }1081        let root = h.tmp.url.path1082        await h.tools.register("test.write", sideEffect: .write, tier: .reversibleWrite, idempotent: false) { call, _ in1083            let path = call.arguments["path"]?.stringValue ?? ""1084            do {1085                try FileManager.default.createDirectory(atPath: (path as NSString).deletingLastPathComponent, withIntermediateDirectories: true)1086                try (call.arguments["content"]?.stringValue ?? "").write(toFile: path, atomically: true, encoding: .utf8)1087                return .success(["path": .string(path)], summary: "wrote")1088            } catch { return .failure(.failed, ToolFailure(code: "io", message: "\(error)")) }1089        }1090        let doc = try workflow("""1091            - id: write1092              tool: test.write1093              with: {path: "${{ params.dir }}/out/report.md", content: "# Weekly report\\nAll good.\\n"}1094            - id: empty1095              tool: test.write1096              with: {path: "${{ params.dir }}/out/empty.txt", content: ""}1097            - id: check1098              needs: [write, empty]1099              verify:1100                - fileExists: ${{ params.dir }}/out/report.md1101                - fileNonEmpty: ${{ steps.write.output.path }}1102                - fileContains: ${{ params.dir }}/out/report.md1103                  text: Weekly report1104                - dirNonEmpty: ${{ params.dir }}/out1105                - equals: ["${{ basename(steps.write.output.path) }}", report.md]1106                - truthy: ${{ endsWith(steps.write.output.path, ".md") }}1107                - stepSucceeded: write1108            - id: out1109              needs: [check]1110              output: {ok: true}1111              artifacts: ["${{ params.dir }}/out/report.md"]11121113        """, spec: "  parameters:\n    - {name: dir, type: path, required: true}\n  permissions:\n    filesystem:\n      read: [\"${{ params.dir }}\"]\n")1114        let outcome = await h.engine().run(doc, options: h.options(["dir": .string(root)]))1115        #expect(outcome.status == .succeeded)1116        #expect(outcome.verifications.count == 7 && outcome.verifications.allSatisfy(\.passed))1117        #expect(outcome.verifications.map(\.kind) == ["file_exists", "file_nonempty", "file_contains", "dir_nonempty", "custom", "custom", "tool_succeeded"])1118        #expect(outcome.artifacts.count == 1 && outcome.artifacts[0].bytes == 26 && outcome.artifacts[0].path.hasSuffix("/out/report.md"))1119        #expect(await h.sink.events(of: .verification).count == 7)11201121        var failing = doc1122        failing.spec.steps[2].kind = .verify(VerifyStep(assertions: [1123            .fileNonEmpty(path: "${{ params.dir }}/out/empty.txt"), .fileExists(path: "${{ params.dir }}/out/missing.md"),1124            .fileContains(path: "${{ params.dir }}/out/report.md", text: "Monthly"), .fileExists(path: "/etc/hosts"),1125            .fileExists(path: "~/.ssh/id_ed25519"), .stepSucceeded("nope"),1126        ]))1127        let h2 = try Harness(); defer { h2.tmp.remove() }1128        await h2.tools.register("test.write") { call, ctx in .success(call.arguments, summary: "pretend") }1129        let bad = await h.engine().run(failing, options: h.options(["dir": .string(root)]))1130        #expect(bad.status == .failed, "SUCCEEDED requires every executed verify to pass")1131        #expect(bad.verifications.map(\.passed) == [false, false, false, false, false, false])1132        #expect(bad.verifications[0].evidence == "0 bytes")1133        #expect(bad.verifications[3].evidence.contains("outside the workflow's declared filesystem permissions"))1134        #expect(bad.verifications[4].evidence.contains("outside") || bad.verifications[4].evidence.contains("protected"))1135        #expect(bad.step("check")?.errorCode == "verification_failed" && bad.step("out") == nil)1136    }1137}11381139@Suite("Engine subworkflows and steps (mocked tools)") struct EngineCompositionTests {1140    @Test func subworkflowReceivesInputsAndReturnsOutputs() async throws {1141        let h = try Harness(); defer { h.tmp.remove() }1142        await h.registerBasics()1143        let child = try WorkflowCodec.parse("""1144        apiVersion: orix.ai/v11145        kind: Automation1146        metadata: {id: child, name: Child}1147        spec:1148          parameters:1149            - {name: text, type: string, required: true}1150            - {name: times, type: integer, default: 2}1151          variables:1152            prefix: "${{ params.text }}:"1153          steps:1154            - id: echo1155              tool: test.echo1156              with: {text: "${{ vars.prefix }}${{ params.times }}"}1157            - id: result1158              needs: [echo]1159              output: {shout: "${{ upper(steps.echo.output.text) }}"}1160        """)1161        let parent = try workflow("""1162            - id: echo1163              tool: test.echo1164              with: {text: parent}1165            - id: call1166              needs: [echo]1167              subworkflow: child1168              inputs: {text: "${{ steps.echo.output.text }}"}1169            - id: out1170              needs: [call]1171              output: {fromChild: "${{ steps.call.output.shout }}", mine: "${{ steps.echo.output.text }}"}11721173        """)1174        let outcome = await h.engine(loader: DictionarySubworkflowLoader([child])).run(parent, options: h.options())1175        #expect(outcome.status == .succeeded && outcome.outputs == ["fromChild": "PARENT:2", "mine": "parent"])1176        #expect(outcome.results(for: "echo").map(\.key) == ["echo", "call>/echo"], "same step ids live in separate namespaces")11771178        let h2 = try Harness(); defer { h2.tmp.remove() }1179        let missing = await h2.engine(loader: DictionarySubworkflowLoader([])).run(parent, options: h2.options())1180        #expect(missing.status == .failed)1181        await h2.registerBasics()1182        let h3 = try Harness(); defer { h3.tmp.remove() }1183        await h3.registerBasics()1184        let noLoader = await h3.engine().run(parent, options: h3.options())1185        #expect(noLoader.step("call")?.error?.contains("no subworkflow loader") == true)1186    }11871188    @Test func recursionIsStoppedByTheDepthLimit() async throws {1189        let h = try Harness(); defer { h.tmp.remove() }1190        let recursive = try WorkflowCodec.parse("""1191        apiVersion: orix.ai/v11192        kind: Automation1193        metadata: {id: narcissus, name: Narcissus}1194        spec:1195          steps:1196            - id: again1197              subworkflow: narcissus1198        """)1199        let outcome = await h.engine(loader: DictionarySubworkflowLoader([recursive]), configuration: .init(maxSubworkflowDepth: 4))1200            .run(recursive, options: h.options())1201        #expect(outcome.status == .failed)1202        #expect(outcome.results(for: "again").count == 5, "four nested calls, the fifth is refused")1203        #expect(outcome.errors.joined().contains("nested deeper than 4 levels"))1204    }12051206    @Test func notifySetTransformAndSecretPlaceholders() async throws {1207        let h = try Harness(); defer { h.tmp.remove() }1208        await h.registerBasics()1209        let doc = try workflow("""1210            - id: compute1211              transform: ${{ sum(map(params.items, x => x.size)) }}1212              into: total1213            - id: login1214              needs: [compute]1215              tool: test.echo1216              with: {password: "${{ secret(\\"smtp\\") }}", total: "${{ vars.total }}"}1217            - id: tell1218              needs: [login]1219              notify: Total is ${{ vars.total }}1220              title: Sizes12211222        """, spec: "  parameters:\n    - name: items\n      type: array\n      default: [{size: 2}, {size: 40}]\n")1223        let outcome = await h.engine().run(doc, options: h.options())1224        #expect(outcome.status == .succeeded)1225        #expect(outcome.step("compute")?.output == 42)1226        #expect(outcome.step("login")?.output == ["password": ["$secret": "smtp"], "total": 42], "a secret is a reference placeholder, never a value")1227        #expect(await h.calls("notifications.post").first?.arguments == ["title": "Sizes", "body": "Total is 42"])1228        let call = try #require(await h.calls("test.echo").first)1229        #expect(call.idempotencyKey == "run-1/login" && !call.dryRun)1230    }1231}1232
Files main
Menu

Theme