SPB Git

Repository Public MIT

AgentBudgetAndLifecycleTests.swift 35.4 KB 575 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Tests/AstraAgentsTests/AgentBudgetAndLifecycleTests.swift10// Module           : AstraAgentsTests11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import Foundation89import Testing90import AstraCore91@testable import AstraAgents9293/// Distinct arguments on every turn, so budgets (not cycle detection) end these runs.94private func endless(_ tool: String = "search", usage: TokenUsage? = TokenUsage(inputTokens: 1_000, outputTokens: 200)) -> ScriptedProvider {95    ScriptedProvider([], fallback: { n in ScriptedProvider.calls([ToolCall(id: "c\(n)", name: tool, arguments: ["q": .string("topic \(n)")])], text: "step \(n) ", usage: usage) })96}9798@Suite struct AgentBudgetTests {99    @Test func stepBudget() async {100        let tool = SpyTool("search")101        let provider = endless()102        let events = await collect(makeRunner(provider, tools: [tool]).run(profile: makeProfile(tools: ["search"], budgets: AgentBudgets(maxSteps: 4)), goal: "g"))103        #expect(events.status == .budgetExceeded(.steps))104        #expect(events.steps == [1, 2, 3, 4])105        #expect(provider.requests.count == 4 && tool.calls.count == 4)106        #expect(events.finalText == "step 4 ")107        #expect(events.warnings.contains { $0.kind == .steps && $0.message == "4 of 4 steps used." })108        #expect(events.warnings.filter { $0.kind == .steps }.count == 1)                 // warned once109    }110111    @Test func toolCallBudget() async {112        let tool = SpyTool("search")113        let three = (1...3).map { ToolCall(id: "c\($0)", name: "search", arguments: ["q": .string("t\($0)")]) }114        let provider = ScriptedProvider([ScriptedProvider.calls(three), ScriptedProvider.calls(three.map { ToolCall(id: $0.id + "b", name: "search", arguments: ["q": .string($0.id)]) }), ScriptedProvider.text("never")])115        let events = await collect(makeRunner(provider, tools: [tool]).run(profile: makeProfile(tools: ["search"], budgets: AgentBudgets(maxToolCalls: 5)), goal: "g"))116        #expect(events.status == .budgetExceeded(.toolCalls))117        #expect(tool.calls.count == 5)                                    // the sixth call was not executed118        #expect(events.warnings.contains { $0.kind == .toolCalls })119        #expect(events.requested.count == 5)120    }121122    @Test func refusedCallsCountAgainstTheToolBudget() async {123        let provider = ScriptedProvider([], fallback: { n in ScriptedProvider.call("ghost_\(n)") })124        let events = await collect(makeRunner(provider).run(profile: makeProfile(budgets: AgentBudgets(maxSteps: 50, maxToolCalls: 3)), goal: "g"))125        #expect(events.status == .budgetExceeded(.toolCalls))126        #expect(events.finished.count == 3)127    }128129    @Test func tokenBudget() async {130        let provider = endless()131        let budgets = AgentBudgets(maxSteps: 50, maxTotalTokens: 3_000)132        let events = await collect(makeRunner(provider, tools: [SpyTool("search")]).run(profile: makeProfile(tools: ["search"], budgets: budgets), goal: "g"))133        #expect(events.status == .budgetExceeded(.tokens))134        #expect(events.steps == [1, 2, 3])                                // 1 200 tokens per turn: 3 600 ≥ 3 000 after the third135        #expect(events.totals.last == TokenUsage(inputTokens: 3_000, outputTokens: 600))136        #expect(events.warnings.contains { $0.kind == .tokens && $0.message == "2400 of 3000 tokens used." })137    }138139    @Test func costBudgetFromPublishedPricing() async {140        let provider = endless()141        let pricing = Pricing(inputPerMTok: 10, outputPerMTok: 50, asOf: "2026-11-01")          // $0.02 per turn142        let budgets = AgentBudgets(maxSteps: 50, maxTotalCostUSD: 0.05)143        let events = await collect(makeRunner(provider, tools: [SpyTool("search")])144            .run(profile: makeProfile(tools: ["search"], budgets: budgets), goal: "g", pricing: pricing))145        #expect(events.status == .budgetExceeded(.cost))146        #expect(events.steps == [1, 2, 3])147        let last = events.costs.last148        #expect(last?.basis == .estimatedFromUsage && last?.pricingAsOf == "2026-11-01")149        #expect(abs((last?.amountUSD ?? 0) - 0.06) < 1e-9)150        #expect(events.warnings.contains { $0.kind == .cost && $0.message.contains("of $0.050 budget used") })151    }152153    @Test func costBudgetFromProviderReportedCost() async {154        let provider = endless(usage: TokenUsage(inputTokens: 10, outputTokens: 5, reportedCostUSD: 0.4))155        let events = await collect(makeRunner(provider, tools: [SpyTool("search")])156            .run(profile: makeProfile(tools: ["search"], budgets: AgentBudgets(maxSteps: 50, maxTotalCostUSD: 1)), goal: "g"))157        #expect(events.status == .budgetExceeded(.cost))158        #expect(events.steps == [1, 2, 3])159        #expect(events.costs.last?.basis == .reportedByProvider)160        #expect(abs((events.costs.last?.amountUSD ?? 0) - 1.2) < 1e-9)161    }162163    @Test func unknownCostIsReportedAsUnknownAndNeverEnforcedAsZero() async {164        // No pricing and no provider-reported cost: the cost cannot be known.165        let provider = ScriptedProvider([ScriptedProvider.call("search", ["q": "a"]), ScriptedProvider.call("search", ["q": "b"]), ScriptedProvider.text("done")])166        let events = await collect(makeRunner(provider, tools: [SpyTool("search")])167            .run(profile: makeProfile(tools: ["search"], budgets: AgentBudgets(maxTotalCostUSD: 0.000_001)), goal: "g"))168        #expect(events.status == .completed)                              // not stopped: nothing was known to exceed169        #expect(events.costs.count == 3)170        #expect(events.costs.allSatisfy { $0 == .unavailable && $0.amountUSD == nil })171        #expect(events.totals.last?.inputTokens == 300)                   // tokens are still real172        let unknown = events.warnings.filter { $0.kind == .cost }173        #expect(unknown.count == 1 && unknown[0].message.contains("cost limit cannot be enforced"))174        #expect(events.checkpoints.last?.cost.unknownTurns == 3)175    }176177    @Test func aTurnWithoutUsageMakesTheTotalUnknown() async {178        let pricing = Pricing(inputPerMTok: 10, outputPerMTok: 50, asOf: "2026-11-01")179        let provider = ScriptedProvider([180            ScriptedProvider.call("search", ["q": "a"]),181            ScriptedProvider.calls([ToolCall(id: "x", name: "search", arguments: ["q": "b"])], usage: nil),182            ScriptedProvider.text("done"),183        ])184        let events = await collect(makeRunner(provider, tools: [SpyTool("search")]).run(profile: makeProfile(tools: ["search"]), goal: "g", pricing: pricing))185        #expect(events.costs.map(\.basis) == [.estimatedFromUsage, .unavailable, .unavailable])186        #expect(events.costs[0].amountUSD != nil && events.costs[2].amountUSD == nil)187        let ledger = events.checkpoints.last?.cost188        #expect(ledger?.knownTurns == 2 && ledger?.unknownTurns == 1 && ledger?.isComplete == false)189        #expect(abs((ledger?.knownUSD ?? 0) - 0.004) < 1e-9)             // the known part stays available as a lower bound190    }191192    @Test func onDeviceRunsCostNothingAndSaySo() async {193        let local = ModelRef(provider: .mlx, modelID: "qwen-local")194        let provider = ScriptedProvider(kind: .mlx, [ScriptedProvider.text("hi")])195        let events = await collect(makeRunner(provider).run(profile: makeProfile(budgets: AgentBudgets(maxTotalCostUSD: 0.01), model: local), goal: "g"))196        #expect(events.status == .completed)197        #expect(events.costs == [.onDevice])198        #expect(events.warnings.isEmpty)199    }200201    @Test func wallClockBudgetDuringTheModelTurn() async {202        let provider = ScriptedProvider([.hang(after: [.messageStarted(MessageStart()), .textDelta("thinking")])])203        let start = Date()204        let events = await collect(makeRunner(provider).run(profile: makeProfile(budgets: AgentBudgets(wallClockLimit: 0.2)), goal: "g"))205        #expect(events.status == .budgetExceeded(.wallClock))206        #expect(events.finalText == "thinking")207        #expect(Date().timeIntervalSince(start) < 5)208    }209210    @Test func wallClockBudgetDuringATool() async {211        let provider = ScriptedProvider([ScriptedProvider.call("hang")])212        let events = await collect(makeRunner(provider, tools: [HangingTool(honoursCancellation: false)])213            .run(profile: makeProfile(tools: ["hang"], budgets: AgentBudgets(wallClockLimit: 0.2)), goal: "g"))214        #expect(events.status == .budgetExceeded(.wallClock))215        #expect(events.started == ["hang"] && events.finished.isEmpty)216    }217218    @Test func wallClockAlreadySpentBeforeAResume() async {219        let first = await collect(makeRunner(ScriptedProvider([ScriptedProvider.call("search", ["q": "a"]), .fail(.timeout)]), tools: [SpyTool("search")])220            .run(profile: makeProfile(tools: ["search"]), goal: "g"))221        var checkpoint = first.checkpoints[0]222        checkpoint.elapsed = 120223        let provider = ScriptedProvider([ScriptedProvider.text("late")])224        let events = await collect(makeRunner(provider, tools: [SpyTool("search")])225            .run(profile: makeProfile(tools: ["search"], budgets: AgentBudgets(wallClockLimit: 60)), goal: "", resumeFrom: checkpoint))226        #expect(events.status == .budgetExceeded(.wallClock))227        #expect(provider.requests.isEmpty)228    }229230    @Test func outputTokenCapReachesEveryRequest() async {231        let provider = ScriptedProvider([ScriptedProvider.call("search", ["q": "a"]), ScriptedProvider.text("done")])232        var profile = makeProfile(tools: ["search"], budgets: AgentBudgets(maxOutputTokensPerTurn: 321))233        profile.parameters = GenerationParameters(temperature: 0.2, maxOutputTokens: 9_999)234        _ = await collect(makeRunner(provider, tools: [SpyTool("search")]).run(profile: profile, goal: "g"))235        #expect(provider.requests.map(\.parameters.maxOutputTokens) == [321, 321])236        #expect(provider.requests[0].parameters.temperature == 0.2)237        profile.budgets.maxOutputTokensPerTurn = 50_000238        let second = ScriptedProvider([ScriptedProvider.text("done")])239        _ = await collect(makeRunner(second).run(profile: profile, goal: "g"))240        #expect(second.requests[0].parameters.maxOutputTokens == 9_999)                  // the lower of the two limits wins241    }242243    @Test func budgetsAreClampedToSaneValues() {244        let budgets = AgentBudgets(maxSteps: 0, maxToolCalls: -5, maxToolOutputCharacters: 1, repeatLimit: 0, alternationLimit: 1, warningFraction: 7)245        #expect(budgets.maxSteps == 1 && budgets.maxToolCalls == 0 && budgets.maxToolOutputCharacters == 256)246        #expect(budgets.repeatLimit == 2 && budgets.alternationLimit == 2 && budgets.warningFraction == 1)247    }248249    @Test func contextWindowDropsOldTurnsButKeepsTheGoal() async {250        let big = SpyTool("search") { _ in String(repeating: "lorem ipsum ", count: 400) }251        let provider = endless()252        var profile = makeProfile(tools: ["search"], budgets: AgentBudgets(maxSteps: 6))253        profile.contextWindow = 6_000254        _ = await collect(makeRunner(provider, tools: [big]).run(profile: profile, goal: "The goal."))255        let last = provider.requests[5].messages256        #expect(last.first == .user("The goal."))257        #expect(last.count < 11)                                          // 1 + 5 × 2 messages without truncation258        #expect(provider.requests[1].messages.count == 3)259        // tool results never lose the call they answer260        for (i, message) in last.enumerated() where message.role == .tool {261            #expect(i > 0 && last[max(0, i - 1)].toolCalls.contains { $0.id == message.toolCallID })262        }263        #expect(last.last?.role == .tool && last.last?.toolCallID == "c5")             // the latest exchange is always kept264        // Earlier conversation is dropped before this run's own exchanges.265        let history: [PromptMessage] = [.user(String(repeating: "old question ", count: 800)), .assistant(String(repeating: "old answer ", count: 800))]266        let fitted = AgentSession.fit(history + [.user("goal"), .assistant("", toolCalls: [ToolCall(id: "1", name: "t", arguments: [:])]), .tool("r", callID: "1", name: "t")], inputBudget: 300)267        #expect(fitted.map(\.role) == [.user, .assistant, .tool] && fitted[0].text == "goal")268        #expect(AgentSession.fit(history, inputBudget: 1_000_000) == history)269    }270}271272// MARK: - Cancellation273274@Suite struct AgentCancellationTests {275    /// Cancels the run the first time `trigger` matches, then returns the whole timeline.276    func cancelling(_ run: AgentRun, when trigger: (AgentEvent) -> Bool) async -> [AgentEvent] {277        var events: [AgentEvent] = []278        var cancelled = false279        for await event in run.events {280            events.append(event)281            if !cancelled, trigger(event) { cancelled = true; run.cancel() }282        }283        return events284    }285286    @Test func duringModelStreaming() async {287        let provider = ScriptedProvider([.hang(after: [.messageStarted(MessageStart()), .textDelta("Partial ans")])])288        let run = makeRunner(provider).start(profile: makeProfile(), goal: "g")289        let start = Date()290        let events = await cancelling(run) { if case .modelTextDelta = $0 { true } else { false } }291        #expect(events.status == .stopped)292        #expect(events.finalText == "Partial ans")293        guard case .runFinished = events.last else { Issue.record("the timeline must end with runFinished"); return }294        #expect(events.checkpoints.isEmpty)295        #expect(Date().timeIntervalSince(start) < 5)296    }297298    @Test(arguments: [true, false])299    func duringATool(honoursCancellation: Bool) async {300        let provider = ScriptedProvider([ScriptedProvider.call("hang"), ScriptedProvider.text("never")])301        let run = makeRunner(provider, tools: [HangingTool(honoursCancellation: honoursCancellation)]).start(profile: makeProfile(tools: ["hang"]), goal: "g")302        let start = Date()303        let events = await cancelling(run) { if case .toolStarted = $0 { true } else { false } }304        #expect(events.status == .stopped)305        #expect(events.started == ["hang"] && events.finished.isEmpty)306        #expect(provider.requests.count == 1)                             // no further model turn307        #expect(Date().timeIntervalSince(start) < 5)308    }309310    @Test func whileWaitingForApproval() async {311        let writer = SpyTool("write_note", risk: .writesUserData)312        let provider = ScriptedProvider([ScriptedProvider.call("write_note", ["text": "x"]), ScriptedProvider.text("never")])313        // A sheet nobody answers.314        let approvals = ApprovalRecorder { _, _ in315            await withCheckedContinuation { (_: CheckedContinuation<Void, Never>) in }316            return .allowOnce317        }318        let run = makeRunner(provider, tools: [writer], approvals: approvals).start(profile: makeProfile(tools: ["write_note"]), goal: "g")319        let start = Date()320        let events = await cancelling(run) { if case .approvalRequested = $0 { true } else { false } }321        #expect(events.status == .stopped)322        #expect(writer.calls.isEmpty)323        #expect(events.decisions.isEmpty && events.started.isEmpty)324        #expect(Date().timeIntervalSince(start) < 5)325    }326327    @Test func aLateApprovalAfterCancellationDoesNotRunTheTool() async {328        let writer = SpyTool("write_note", risk: .writesUserData)329        let provider = ScriptedProvider([ScriptedProvider.call("write_note", ["text": "x"])])330        let approvals = ApprovalRecorder { _, _ in331            try? await Task.sleep(nanoseconds: 300_000_000)                // answers after the run was stopped332            return .allowForRun333        }334        let run = makeRunner(provider, tools: [writer], approvals: approvals).start(profile: makeProfile(tools: ["write_note"]), goal: "g")335        let events = await cancelling(run) { if case .approvalRequested = $0 { true } else { false } }336        try? await Task.sleep(nanoseconds: 500_000_000)337        #expect(events.status == .stopped && writer.calls.isEmpty)338    }339340    @Test func betweenSteps() async {341        let tool = SpyTool("search")342        let run = makeRunner(endless(), tools: [tool]).start(profile: makeProfile(tools: ["search"], budgets: AgentBudgets(maxSteps: 1_000, maxToolCalls: 10_000)), goal: "g")343        let events = await cancelling(run) { if case .checkpointSaved(let c) = $0 { c.step == 3 } else { false } }344        #expect(events.status == .stopped)345        #expect(events.steps.count >= 3 && events.steps.count <= 5)346        #expect(events.checkpoints.count >= 3)                            // the work done so far can be resumed347    }348349    @Test func beforeTheFirstEventIsRead() async {350        let provider = ScriptedProvider([.hang(after: [])])351        let run = makeRunner(provider).start(profile: makeProfile(), goal: "g")352        run.cancel()353        let events = await collect(run.events)354        #expect(events.status == .stopped)355        #expect(events.filter { if case .runFinished = $0 { true } else { false } }.count == 1)356    }357358    @Test func droppingTheStreamStopsTheWork() async {359        let tool = SpyTool("search")360        let provider = endless()361        do {362            let stream = makeRunner(provider, tools: [tool]).run(profile: makeProfile(tools: ["search"], budgets: AgentBudgets(maxSteps: 100_000, maxToolCalls: 100_000)), goal: "g")363            for await event in stream { if case .stepStarted(3) = event { break } }364        }365        try? await Task.sleep(nanoseconds: 200_000_000)366        let settled = provider.requests.count367        try? await Task.sleep(nanoseconds: 200_000_000)368        #expect(provider.requests.count == settled)                       // nothing keeps running in the background369        #expect(settled < 50)370    }371}372373// MARK: - Checkpoints374375@Suite struct AgentCheckpointTests {376    @Test func roundTripsThroughJSON() async throws {377        let writer = SpyTool("write_note", risk: .writesUserData)378        let provider = ScriptedProvider([379            ScriptedProvider.call("update_plan", ["steps": [["title": "Write", "status": "in_progress"], ["title": "Report", "status": "pending"]]]),380            ScriptedProvider.calls([ToolCall(id: "w1", name: "write_note", arguments: ["text": "héllo ✓", "n": 1.5])], text: "Saving."),381            .fail(.timeout),382        ])383        let events = await collect(makeRunner(provider, tools: [writer], approvals: ApprovalRecorder(always: .allowForRun))384            .run(profile: makeProfile(tools: ["write_note"]), goal: "Save a note.", pricing: Pricing(inputPerMTok: 3, outputPerMTok: 15, asOf: "2026-11-01")))385        #expect(events.status == .failed(AstraError.timeout.message))386        #expect(events.checkpoints.map(\.step) == [1, 2])                 // one per completed step387        let checkpoint = try #require(events.checkpoints.last)388        let data = try checkpoint.encoded()389        let decoded = try AgentCheckpoint.decode(data)390        #expect(decoded == checkpoint)391        #expect(try decoded.encoded() == data)                            // stable encoding392        #expect(decoded.version == AgentCheckpoint.currentVersion)393        #expect(decoded.goal == "Save a note." && decoded.profileID == "profile-1")394        #expect(decoded.step == 2 && decoded.toolCalls == 1 && !decoded.isComplete)395        #expect(decoded.messages.count == 5)                              // goal, 2 assistant turns, 2 tool results396        #expect(decoded.messages[3].toolCalls == [ToolCall(id: "w1", name: "write_note", arguments: ["text": "héllo ✓", "n": 1.5])])397        #expect(decoded.plan?.steps.map(\.title) == ["Write", "Report"])398        #expect(decoded.grants.forRun == ["write_note"])399        #expect(decoded.usage == TokenUsage(inputTokens: 200, outputTokens: 40))400        #expect(decoded.cost.knownTurns == 2 && decoded.cost.isComplete)401        #expect(decoded.recentCalls.count == 1)402    }403404    @Test func newerVersionsAreRefused() async throws {405        let events = await collect(makeRunner(ScriptedProvider([ScriptedProvider.text("hi")])).run(profile: makeProfile(), goal: "g"))406        var checkpoint = try #require(events.checkpoints.last)407        checkpoint.version = AgentCheckpoint.currentVersion + 1408        let data = try checkpoint.encoded()409        #expect(throws: AstraError.self) { try AgentCheckpoint.decode(data) }410        #expect(throws: (any Error).self) { try AgentCheckpoint.decode(Data("{}".utf8)) }411    }412413    @Test func resumesAfterARelaunch() async throws {414        // First launch: one completed step, then the app "dies" (the provider fails on step 2).415        let writer = SpyTool("write_note", risk: .writesUserData)416        let firstProvider = ScriptedProvider([417            ScriptedProvider.calls([ToolCall(id: "p1", name: "update_plan", arguments: ["steps": [["title": "Draft", "status": "in_progress"]]]),418                                    ToolCall(id: "w1", name: "write_note", arguments: ["text": "draft 1"])]),419            .fail(.network("The connection was lost.")),420        ])421        let firstApprovals = ApprovalRecorder(always: .allowForRun)422        let firstRun = makeRunner(firstProvider, tools: [writer], approvals: firstApprovals).start(profile: makeProfile(tools: ["write_note"]), goal: "Write two notes.")423        let first = await collect(firstRun.events)424        guard case .failed = first.status else { Issue.record("expected the first run to fail"); return }425        let saved = try #require(first.checkpoints.last).encoded()        // what the app persisted426427        // Second launch: everything is rebuilt from scratch; only the bytes survive.428        let restored = try AgentCheckpoint.decode(saved)429        let writerAgain = SpyTool("write_note", risk: .writesUserData)430        let secondProvider = ScriptedProvider([ScriptedProvider.call("write_note", ["text": "draft 2"], id: "w2"), ScriptedProvider.text("Both notes are saved.")])431        let secondApprovals = ApprovalRecorder(always: .deny(reason: "should not be asked"))432        let secondRun = makeRunner(secondProvider, tools: [writerAgain], approvals: secondApprovals)433            .start(profile: makeProfile(tools: ["write_note"]), goal: "ignored on resume", resumeFrom: restored)434        let second = await collect(secondRun.events)435436        #expect(secondRun.id == firstRun.id)                              // same run437        #expect(second.status == .completed && second.finalText == "Both notes are saved.")438        guard case .runStarted(let info) = second.first else { Issue.record("runStarted first"); return }439        #expect(info.resumedAfterStep == 1 && info.goal == "Write two notes." && info.runID == firstRun.id)440        #expect(second.steps == [2, 3])                                   // counters continue441        #expect(second.plans.first?.steps.map(\.title) == ["Draft"])      // the plan is restored and re-announced442        #expect(secondApprovals.requests.isEmpty)                         // the "allow for this run" grant survived443        #expect(writerAgain.calls == [["text": "draft 2"]])444        // The model sees the whole history, including the first launch's tool result.445        let history = secondProvider.requests[0].messages446        #expect(history.first == .user("Write two notes."))447        #expect(history.count == 4 && history[2].toolCallID == "p1" && history[3].toolCallID == "w1")448        #expect(history[3].text.contains(UntrustedEnvelope(runID: firstRun.id).closeMarker))      // same envelope nonce across the resume449        let final = try #require(second.checkpoints.last)450        #expect(final.isComplete && final.step == 3 && final.toolCalls == 2)451        #expect(final.usage == TokenUsage(inputTokens: 300, outputTokens: 60))        // usage accumulates across launches452    }453454    @Test func resumeKeepsBudgetsAndCycleHistory() async throws {455        let tool = SpyTool("search")456        let first = await collect(makeRunner(ScriptedProvider([ScriptedProvider.call("search", ["q": "x"]), ScriptedProvider.call("search", ["q": "x"]), .fail(.timeout)]), tools: [tool])457            .run(profile: makeProfile(tools: ["search"]), goal: "g"))458        let checkpoint = try AgentCheckpoint.decode(try #require(first.checkpoints.last).encoded())459        // The third identical call arrives after the relaunch: still a cycle.460        let provider = ScriptedProvider([ScriptedProvider.call("search", ["q": "x"]), ScriptedProvider.text("never")])461        let resumed = await collect(makeRunner(provider, tools: [tool]).run(profile: makeProfile(tools: ["search"]), goal: "", resumeFrom: checkpoint))462        guard case .cycleDetected = resumed.status else { Issue.record("expected a cycle, got \(String(describing: resumed.status))"); return }463        #expect(tool.calls.count == 2)464        // And the step budget counts the steps of the first launch.465        let capped = await collect(makeRunner(ScriptedProvider([ScriptedProvider.text("never")]), tools: [tool])466            .run(profile: makeProfile(tools: ["search"], budgets: AgentBudgets(maxSteps: 2)), goal: "", resumeFrom: checkpoint))467        #expect(capped.status == .budgetExceeded(.steps))468    }469470    @Test func resumeUnderANarrowerProfileDropsToolsAndGrants() async throws {471        let writer = SpyTool("write_note", risk: .writesUserData)472        let first = await collect(makeRunner(ScriptedProvider([ScriptedProvider.call("write_note", ["text": "a"]), .fail(.timeout)]), tools: [writer], approvals: ApprovalRecorder(always: .allowForRun))473            .run(profile: makeProfile(tools: ["write_note"]), goal: "g"))474        var checkpoint = try #require(first.checkpoints.last)475        #expect(checkpoint.grants.forRun == ["write_note"])476        // Even a tampered checkpoint cannot add permissions: the profile decides.477        checkpoint.grants = ApprovalGrants(forRun: ["write_note", "delete_everything", "web_fetch|evil.example"], readsApproved: ["read_file"])478        let provider = ScriptedProvider([ScriptedProvider.call("write_note", ["text": "b"]), ScriptedProvider.text("done")])479        let approvals = ApprovalRecorder(always: .allowOnce)480        let resumed = await collect(makeRunner(provider, tools: [writer, SpyTool("calculator2")], approvals: approvals)481            .run(profile: makeProfile(tools: ["calculator2"]), goal: "", resumeFrom: checkpoint))482        #expect(resumed.status == .completed)483        #expect(writer.calls.count == 1)                                  // only the call from the first launch484        #expect(resumed.finished[0].isError && resumed.finished[0].output.contains("not available to this agent"))485        #expect(resumed.checkpoints.last?.grants == ApprovalGrants())486        #expect(approvals.requests.isEmpty)487    }488489    @Test func resumingAFinishedRunJustReportsItsAnswer() async throws {490        let done = await collect(makeRunner(ScriptedProvider([ScriptedProvider.text("Final answer.")])).run(profile: makeProfile(), goal: "g"))491        let checkpoint = try #require(done.checkpoints.last)492        let provider = ScriptedProvider([ScriptedProvider.text("should not be asked")])493        let again = await collect(makeRunner(provider).run(profile: makeProfile(), goal: "", resumeFrom: checkpoint))494        #expect(again.status == .completed && again.finalText == "Final answer.")495        #expect(provider.requests.isEmpty && again.steps.isEmpty)496    }497}498499// MARK: - Audit500501@Suite struct AgentAuditTests {502    @Test func secretsAreMaskedInEveryAuditEvent() async {503        let api = SpyTool("call_api", risk: .sendsDataExternally) { _ in504            "status: ok\nAuthorization: Bearer abcdefghijklmnopqrstuvwxyz\nsession_token=ZZZZ99998888\nbody: hello sk-ant-api03-SECRETSECRET12345"505        }506        let arguments: JSONValue = [507            "endpoint": "https://api.example.com/v1/items?api_key=QQQQ77776666", "api_key": "sk-proj-abcdefgh12345678", "password": "hunter2!",508            "headers": ["Authorization": "Bearer abcdefghijklmnopqrstuvwxyz", "Accept": "application/json"], "token": "tok_live_1234567890", "query": "public words",509        ]510        let provider = ScriptedProvider([ScriptedProvider.calls([ToolCall(id: "a1", name: "call_api", arguments: arguments)]), ScriptedProvider.text("done")])511        let approvals = ApprovalRecorder(always: .allowOnce)512        let events = await collect(makeRunner(provider, tools: [api], approvals: approvals).run(profile: makeProfile(tools: ["call_api"]), goal: "g"))513        #expect(events.status == .completed)514515        let secrets = ["QQQQ77776666", "abcdefgh12345678", "hunter2", "abcdefghijklmnopqrstuvwxyz", "tok_live_1234567890", "ZZZZ99998888", "SECRETSECRET12345"]516        // Everything the timeline exposes: tool request, approval request, tool output.517        var audit = ""518        for event in events {519            switch event {520            case .toolRequested(let call): audit += call.arguments.serialized()521            case .approvalRequested(let request): audit += request.summary + request.arguments.serialized() + request.warnings.joined()522            case .toolFinished(_, _, let output, _, _): audit += output523            default: break524            }525        }526        for secret in secrets { #expect(!audit.contains(secret), "\(secret) leaked into the audit timeline") }527        #expect(audit.contains("public words") && audit.contains("application/json") && audit.contains("status: ok"))528529        let requested = events.requested[0].arguments530        #expect(requested["api_key"] == "•••" && requested["password"] == "•••" && requested["token"] == "•••")531        #expect(requested["headers"]?["Authorization"] == "•••" && requested["headers"]?["Accept"] == "application/json")532        #expect(requested["query"] == "public words")533        #expect(approvals.requests[0].arguments["password"] == "•••")534        #expect(approvals.requests[0].warnings.contains { $0.contains("key or a password") })535        // The tool itself received the real arguments: redaction is for the audit trail, not for execution.536        #expect(api.calls == [arguments])537    }538539    @Test func longToolOutputIsTruncatedWithAMarker() async {540        let verbose = SpyTool("dump") { _ in String(repeating: "0123456789", count: 5_000) }541        let provider = ScriptedProvider([ScriptedProvider.call("dump", [:], id: "d1"), ScriptedProvider.text("done")])542        let budgets = AgentBudgets(maxToolOutputCharacters: 1_000)543        let events = await collect(makeRunner(provider, tools: [verbose]).run(profile: makeProfile(tools: ["dump"], budgets: budgets), goal: "g"))544        let output = events.finished[0].output545        #expect(output.hasSuffix("[… output truncated: 1000 of 50000 characters shown]"))546        #expect(output.count < 1_100)547        let delivered = toolMessages(provider, request: 1)[0].text548        #expect(delivered.contains("[… output truncated: 1000 of 50000 characters shown]"))549        #expect(delivered.count < 1_600)                                  // the model never receives the 50 000 characters550        #expect(delivered.hasSuffix(">>>"))                               // and the envelope is still closed551    }552553    @Test func longArgumentsAreShortenedInTheTimeline() async {554        let provider = ScriptedProvider([ScriptedProvider.call("write_file", ["file_name": "big.txt", "content": .string(String(repeating: "word ", count: 20_000))]), ScriptedProvider.text("done")])555        let files = FakeFiles()556        let approvals = ApprovalRecorder(always: .allowOnce)557        let events = await collect(AgentRunner(orchestrator: InferenceOrchestrator(maxRetries: 0) { _ in provider }, registry: .standard(files: files), approvals: approvals)558            .run(profile: makeProfile(tools: ["write_file"]), goal: "g"))559        #expect((events.requested[0].arguments["content"]?.stringValue?.count ?? .max) < 2_100)560        #expect((approvals.requests[0].arguments["content"]?.stringValue?.count ?? .max) < 700)561        #expect(approvals.requests[0].summary.hasPrefix("Save a new file “big.txt” (100000 characters): word word"))562        #expect(files.writes.first?.1.count == 100_000)                   // the file itself is complete563    }564565    @Test func systemPromptStatesTheRulesButIsNotTheEnforcement() {566        let profile = makeProfile(tools: ["calculator"])567        let runner = makeRunner(ScriptedProvider([]), tools: [CalculatorTool()])568        let prompt = AgentRunner.systemPrompt(profile: profile, tools: runner.offeredTools(for: profile))569        #expect(prompt.hasPrefix("Be exact."))570        #expect(prompt.contains("update_plan, calculator") && prompt.contains("ASTRA_TOOL_RESULT") && prompt.contains("You cannot run code"))571        let bare = AgentRunner.systemPrompt(profile: makeProfile(planning: false), tools: [])572        #expect(bare.contains("You have no tools"))573    }574}575
Files main
Menu

Theme