Repository Public MIT
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Astra9// File : Tests/AstraAgentsTests/AgentBudgetAndLifecycleTests.swift10// Module : AstraAgentsTests11// Repository : https://git.spboucher.ai/orix-astra12// Created : 2026-11-2113// Last Modified : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author : Simon-Pierre Boucher20// Position : Professor of Finance21// Institution : Université du Québec en Outaouais (UQO)22// Department : Département des sciences administratives23// Organization : SP Boucher24//25// Website : https://spboucher.ai26// Git : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General : info@spboucher.ai33// Professional : contact@spboucher.ai34// Direct : simon@spboucher.ai35// Research : research@spboucher.ai36// Development : dev@spboucher.ai37// Support : support@spboucher.ai38// Security/Admin : admin@spboucher.ai39// Notifications : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification : Public66// Distribution : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License : Proprietary, all rights reserved (see license file)80// License File : LICENSE81// Maintainer : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import Foundation89import Testing90import AstraCore91@testable import AstraAgents9293/// Distinct arguments on every turn, so budgets (not cycle detection) end these runs.94private func endless(_ tool: String = "search", usage: TokenUsage? = TokenUsage(inputTokens: 1_000, outputTokens: 200)) -> ScriptedProvider {95 ScriptedProvider([], fallback: { n in ScriptedProvider.calls([ToolCall(id: "c\(n)", name: tool, arguments: ["q": .string("topic \(n)")])], text: "step \(n) ", usage: usage) })96}9798@Suite struct AgentBudgetTests {99 @Test func stepBudget() async {100 let tool = SpyTool("search")101 let provider = endless()102 let events = await collect(makeRunner(provider, tools: [tool]).run(profile: makeProfile(tools: ["search"], budgets: AgentBudgets(maxSteps: 4)), goal: "g"))103 #expect(events.status == .budgetExceeded(.steps))104 #expect(events.steps == [1, 2, 3, 4])105 #expect(provider.requests.count == 4 && tool.calls.count == 4)106 #expect(events.finalText == "step 4 ")107 #expect(events.warnings.contains { $0.kind == .steps && $0.message == "4 of 4 steps used." })108 #expect(events.warnings.filter { $0.kind == .steps }.count == 1) // warned once109 }110111 @Test func toolCallBudget() async {112 let tool = SpyTool("search")113 let three = (1...3).map { ToolCall(id: "c\($0)", name: "search", arguments: ["q": .string("t\($0)")]) }114 let provider = ScriptedProvider([ScriptedProvider.calls(three), ScriptedProvider.calls(three.map { ToolCall(id: $0.id + "b", name: "search", arguments: ["q": .string($0.id)]) }), ScriptedProvider.text("never")])115 let events = await collect(makeRunner(provider, tools: [tool]).run(profile: makeProfile(tools: ["search"], budgets: AgentBudgets(maxToolCalls: 5)), goal: "g"))116 #expect(events.status == .budgetExceeded(.toolCalls))117 #expect(tool.calls.count == 5) // the sixth call was not executed118 #expect(events.warnings.contains { $0.kind == .toolCalls })119 #expect(events.requested.count == 5)120 }121122 @Test func refusedCallsCountAgainstTheToolBudget() async {123 let provider = ScriptedProvider([], fallback: { n in ScriptedProvider.call("ghost_\(n)") })124 let events = await collect(makeRunner(provider).run(profile: makeProfile(budgets: AgentBudgets(maxSteps: 50, maxToolCalls: 3)), goal: "g"))125 #expect(events.status == .budgetExceeded(.toolCalls))126 #expect(events.finished.count == 3)127 }128129 @Test func tokenBudget() async {130 let provider = endless()131 let budgets = AgentBudgets(maxSteps: 50, maxTotalTokens: 3_000)132 let events = await collect(makeRunner(provider, tools: [SpyTool("search")]).run(profile: makeProfile(tools: ["search"], budgets: budgets), goal: "g"))133 #expect(events.status == .budgetExceeded(.tokens))134 #expect(events.steps == [1, 2, 3]) // 1 200 tokens per turn: 3 600 ≥ 3 000 after the third135 #expect(events.totals.last == TokenUsage(inputTokens: 3_000, outputTokens: 600))136 #expect(events.warnings.contains { $0.kind == .tokens && $0.message == "2400 of 3000 tokens used." })137 }138139 @Test func costBudgetFromPublishedPricing() async {140 let provider = endless()141 let pricing = Pricing(inputPerMTok: 10, outputPerMTok: 50, asOf: "2026-11-01") // $0.02 per turn142 let budgets = AgentBudgets(maxSteps: 50, maxTotalCostUSD: 0.05)143 let events = await collect(makeRunner(provider, tools: [SpyTool("search")])144 .run(profile: makeProfile(tools: ["search"], budgets: budgets), goal: "g", pricing: pricing))145 #expect(events.status == .budgetExceeded(.cost))146 #expect(events.steps == [1, 2, 3])147 let last = events.costs.last148 #expect(last?.basis == .estimatedFromUsage && last?.pricingAsOf == "2026-11-01")149 #expect(abs((last?.amountUSD ?? 0) - 0.06) < 1e-9)150 #expect(events.warnings.contains { $0.kind == .cost && $0.message.contains("of $0.050 budget used") })151 }152153 @Test func costBudgetFromProviderReportedCost() async {154 let provider = endless(usage: TokenUsage(inputTokens: 10, outputTokens: 5, reportedCostUSD: 0.4))155 let events = await collect(makeRunner(provider, tools: [SpyTool("search")])156 .run(profile: makeProfile(tools: ["search"], budgets: AgentBudgets(maxSteps: 50, maxTotalCostUSD: 1)), goal: "g"))157 #expect(events.status == .budgetExceeded(.cost))158 #expect(events.steps == [1, 2, 3])159 #expect(events.costs.last?.basis == .reportedByProvider)160 #expect(abs((events.costs.last?.amountUSD ?? 0) - 1.2) < 1e-9)161 }162163 @Test func unknownCostIsReportedAsUnknownAndNeverEnforcedAsZero() async {164 // No pricing and no provider-reported cost: the cost cannot be known.165 let provider = ScriptedProvider([ScriptedProvider.call("search", ["q": "a"]), ScriptedProvider.call("search", ["q": "b"]), ScriptedProvider.text("done")])166 let events = await collect(makeRunner(provider, tools: [SpyTool("search")])167 .run(profile: makeProfile(tools: ["search"], budgets: AgentBudgets(maxTotalCostUSD: 0.000_001)), goal: "g"))168 #expect(events.status == .completed) // not stopped: nothing was known to exceed169 #expect(events.costs.count == 3)170 #expect(events.costs.allSatisfy { $0 == .unavailable && $0.amountUSD == nil })171 #expect(events.totals.last?.inputTokens == 300) // tokens are still real172 let unknown = events.warnings.filter { $0.kind == .cost }173 #expect(unknown.count == 1 && unknown[0].message.contains("cost limit cannot be enforced"))174 #expect(events.checkpoints.last?.cost.unknownTurns == 3)175 }176177 @Test func aTurnWithoutUsageMakesTheTotalUnknown() async {178 let pricing = Pricing(inputPerMTok: 10, outputPerMTok: 50, asOf: "2026-11-01")179 let provider = ScriptedProvider([180 ScriptedProvider.call("search", ["q": "a"]),181 ScriptedProvider.calls([ToolCall(id: "x", name: "search", arguments: ["q": "b"])], usage: nil),182 ScriptedProvider.text("done"),183 ])184 let events = await collect(makeRunner(provider, tools: [SpyTool("search")]).run(profile: makeProfile(tools: ["search"]), goal: "g", pricing: pricing))185 #expect(events.costs.map(\.basis) == [.estimatedFromUsage, .unavailable, .unavailable])186 #expect(events.costs[0].amountUSD != nil && events.costs[2].amountUSD == nil)187 let ledger = events.checkpoints.last?.cost188 #expect(ledger?.knownTurns == 2 && ledger?.unknownTurns == 1 && ledger?.isComplete == false)189 #expect(abs((ledger?.knownUSD ?? 0) - 0.004) < 1e-9) // the known part stays available as a lower bound190 }191192 @Test func onDeviceRunsCostNothingAndSaySo() async {193 let local = ModelRef(provider: .mlx, modelID: "qwen-local")194 let provider = ScriptedProvider(kind: .mlx, [ScriptedProvider.text("hi")])195 let events = await collect(makeRunner(provider).run(profile: makeProfile(budgets: AgentBudgets(maxTotalCostUSD: 0.01), model: local), goal: "g"))196 #expect(events.status == .completed)197 #expect(events.costs == [.onDevice])198 #expect(events.warnings.isEmpty)199 }200201 @Test func wallClockBudgetDuringTheModelTurn() async {202 let provider = ScriptedProvider([.hang(after: [.messageStarted(MessageStart()), .textDelta("thinking")])])203 let start = Date()204 let events = await collect(makeRunner(provider).run(profile: makeProfile(budgets: AgentBudgets(wallClockLimit: 0.2)), goal: "g"))205 #expect(events.status == .budgetExceeded(.wallClock))206 #expect(events.finalText == "thinking")207 #expect(Date().timeIntervalSince(start) < 5)208 }209210 @Test func wallClockBudgetDuringATool() async {211 let provider = ScriptedProvider([ScriptedProvider.call("hang")])212 let events = await collect(makeRunner(provider, tools: [HangingTool(honoursCancellation: false)])213 .run(profile: makeProfile(tools: ["hang"], budgets: AgentBudgets(wallClockLimit: 0.2)), goal: "g"))214 #expect(events.status == .budgetExceeded(.wallClock))215 #expect(events.started == ["hang"] && events.finished.isEmpty)216 }217218 @Test func wallClockAlreadySpentBeforeAResume() async {219 let first = await collect(makeRunner(ScriptedProvider([ScriptedProvider.call("search", ["q": "a"]), .fail(.timeout)]), tools: [SpyTool("search")])220 .run(profile: makeProfile(tools: ["search"]), goal: "g"))221 var checkpoint = first.checkpoints[0]222 checkpoint.elapsed = 120223 let provider = ScriptedProvider([ScriptedProvider.text("late")])224 let events = await collect(makeRunner(provider, tools: [SpyTool("search")])225 .run(profile: makeProfile(tools: ["search"], budgets: AgentBudgets(wallClockLimit: 60)), goal: "", resumeFrom: checkpoint))226 #expect(events.status == .budgetExceeded(.wallClock))227 #expect(provider.requests.isEmpty)228 }229230 @Test func outputTokenCapReachesEveryRequest() async {231 let provider = ScriptedProvider([ScriptedProvider.call("search", ["q": "a"]), ScriptedProvider.text("done")])232 var profile = makeProfile(tools: ["search"], budgets: AgentBudgets(maxOutputTokensPerTurn: 321))233 profile.parameters = GenerationParameters(temperature: 0.2, maxOutputTokens: 9_999)234 _ = await collect(makeRunner(provider, tools: [SpyTool("search")]).run(profile: profile, goal: "g"))235 #expect(provider.requests.map(\.parameters.maxOutputTokens) == [321, 321])236 #expect(provider.requests[0].parameters.temperature == 0.2)237 profile.budgets.maxOutputTokensPerTurn = 50_000238 let second = ScriptedProvider([ScriptedProvider.text("done")])239 _ = await collect(makeRunner(second).run(profile: profile, goal: "g"))240 #expect(second.requests[0].parameters.maxOutputTokens == 9_999) // the lower of the two limits wins241 }242243 @Test func budgetsAreClampedToSaneValues() {244 let budgets = AgentBudgets(maxSteps: 0, maxToolCalls: -5, maxToolOutputCharacters: 1, repeatLimit: 0, alternationLimit: 1, warningFraction: 7)245 #expect(budgets.maxSteps == 1 && budgets.maxToolCalls == 0 && budgets.maxToolOutputCharacters == 256)246 #expect(budgets.repeatLimit == 2 && budgets.alternationLimit == 2 && budgets.warningFraction == 1)247 }248249 @Test func contextWindowDropsOldTurnsButKeepsTheGoal() async {250 let big = SpyTool("search") { _ in String(repeating: "lorem ipsum ", count: 400) }251 let provider = endless()252 var profile = makeProfile(tools: ["search"], budgets: AgentBudgets(maxSteps: 6))253 profile.contextWindow = 6_000254 _ = await collect(makeRunner(provider, tools: [big]).run(profile: profile, goal: "The goal."))255 let last = provider.requests[5].messages256 #expect(last.first == .user("The goal."))257 #expect(last.count < 11) // 1 + 5 × 2 messages without truncation258 #expect(provider.requests[1].messages.count == 3)259 // tool results never lose the call they answer260 for (i, message) in last.enumerated() where message.role == .tool {261 #expect(i > 0 && last[max(0, i - 1)].toolCalls.contains { $0.id == message.toolCallID })262 }263 #expect(last.last?.role == .tool && last.last?.toolCallID == "c5") // the latest exchange is always kept264 // Earlier conversation is dropped before this run's own exchanges.265 let history: [PromptMessage] = [.user(String(repeating: "old question ", count: 800)), .assistant(String(repeating: "old answer ", count: 800))]266 let fitted = AgentSession.fit(history + [.user("goal"), .assistant("", toolCalls: [ToolCall(id: "1", name: "t", arguments: [:])]), .tool("r", callID: "1", name: "t")], inputBudget: 300)267 #expect(fitted.map(\.role) == [.user, .assistant, .tool] && fitted[0].text == "goal")268 #expect(AgentSession.fit(history, inputBudget: 1_000_000) == history)269 }270}271272// MARK: - Cancellation273274@Suite struct AgentCancellationTests {275 /// Cancels the run the first time `trigger` matches, then returns the whole timeline.276 func cancelling(_ run: AgentRun, when trigger: (AgentEvent) -> Bool) async -> [AgentEvent] {277 var events: [AgentEvent] = []278 var cancelled = false279 for await event in run.events {280 events.append(event)281 if !cancelled, trigger(event) { cancelled = true; run.cancel() }282 }283 return events284 }285286 @Test func duringModelStreaming() async {287 let provider = ScriptedProvider([.hang(after: [.messageStarted(MessageStart()), .textDelta("Partial ans")])])288 let run = makeRunner(provider).start(profile: makeProfile(), goal: "g")289 let start = Date()290 let events = await cancelling(run) { if case .modelTextDelta = $0 { true } else { false } }291 #expect(events.status == .stopped)292 #expect(events.finalText == "Partial ans")293 guard case .runFinished = events.last else { Issue.record("the timeline must end with runFinished"); return }294 #expect(events.checkpoints.isEmpty)295 #expect(Date().timeIntervalSince(start) < 5)296 }297298 @Test(arguments: [true, false])299 func duringATool(honoursCancellation: Bool) async {300 let provider = ScriptedProvider([ScriptedProvider.call("hang"), ScriptedProvider.text("never")])301 let run = makeRunner(provider, tools: [HangingTool(honoursCancellation: honoursCancellation)]).start(profile: makeProfile(tools: ["hang"]), goal: "g")302 let start = Date()303 let events = await cancelling(run) { if case .toolStarted = $0 { true } else { false } }304 #expect(events.status == .stopped)305 #expect(events.started == ["hang"] && events.finished.isEmpty)306 #expect(provider.requests.count == 1) // no further model turn307 #expect(Date().timeIntervalSince(start) < 5)308 }309310 @Test func whileWaitingForApproval() async {311 let writer = SpyTool("write_note", risk: .writesUserData)312 let provider = ScriptedProvider([ScriptedProvider.call("write_note", ["text": "x"]), ScriptedProvider.text("never")])313 // A sheet nobody answers.314 let approvals = ApprovalRecorder { _, _ in315 await withCheckedContinuation { (_: CheckedContinuation<Void, Never>) in }316 return .allowOnce317 }318 let run = makeRunner(provider, tools: [writer], approvals: approvals).start(profile: makeProfile(tools: ["write_note"]), goal: "g")319 let start = Date()320 let events = await cancelling(run) { if case .approvalRequested = $0 { true } else { false } }321 #expect(events.status == .stopped)322 #expect(writer.calls.isEmpty)323 #expect(events.decisions.isEmpty && events.started.isEmpty)324 #expect(Date().timeIntervalSince(start) < 5)325 }326327 @Test func aLateApprovalAfterCancellationDoesNotRunTheTool() async {328 let writer = SpyTool("write_note", risk: .writesUserData)329 let provider = ScriptedProvider([ScriptedProvider.call("write_note", ["text": "x"])])330 let approvals = ApprovalRecorder { _, _ in331 try? await Task.sleep(nanoseconds: 300_000_000) // answers after the run was stopped332 return .allowForRun333 }334 let run = makeRunner(provider, tools: [writer], approvals: approvals).start(profile: makeProfile(tools: ["write_note"]), goal: "g")335 let events = await cancelling(run) { if case .approvalRequested = $0 { true } else { false } }336 try? await Task.sleep(nanoseconds: 500_000_000)337 #expect(events.status == .stopped && writer.calls.isEmpty)338 }339340 @Test func betweenSteps() async {341 let tool = SpyTool("search")342 let run = makeRunner(endless(), tools: [tool]).start(profile: makeProfile(tools: ["search"], budgets: AgentBudgets(maxSteps: 1_000, maxToolCalls: 10_000)), goal: "g")343 let events = await cancelling(run) { if case .checkpointSaved(let c) = $0 { c.step == 3 } else { false } }344 #expect(events.status == .stopped)345 #expect(events.steps.count >= 3 && events.steps.count <= 5)346 #expect(events.checkpoints.count >= 3) // the work done so far can be resumed347 }348349 @Test func beforeTheFirstEventIsRead() async {350 let provider = ScriptedProvider([.hang(after: [])])351 let run = makeRunner(provider).start(profile: makeProfile(), goal: "g")352 run.cancel()353 let events = await collect(run.events)354 #expect(events.status == .stopped)355 #expect(events.filter { if case .runFinished = $0 { true } else { false } }.count == 1)356 }357358 @Test func droppingTheStreamStopsTheWork() async {359 let tool = SpyTool("search")360 let provider = endless()361 do {362 let stream = makeRunner(provider, tools: [tool]).run(profile: makeProfile(tools: ["search"], budgets: AgentBudgets(maxSteps: 100_000, maxToolCalls: 100_000)), goal: "g")363 for await event in stream { if case .stepStarted(3) = event { break } }364 }365 try? await Task.sleep(nanoseconds: 200_000_000)366 let settled = provider.requests.count367 try? await Task.sleep(nanoseconds: 200_000_000)368 #expect(provider.requests.count == settled) // nothing keeps running in the background369 #expect(settled < 50)370 }371}372373// MARK: - Checkpoints374375@Suite struct AgentCheckpointTests {376 @Test func roundTripsThroughJSON() async throws {377 let writer = SpyTool("write_note", risk: .writesUserData)378 let provider = ScriptedProvider([379 ScriptedProvider.call("update_plan", ["steps": [["title": "Write", "status": "in_progress"], ["title": "Report", "status": "pending"]]]),380 ScriptedProvider.calls([ToolCall(id: "w1", name: "write_note", arguments: ["text": "héllo ✓", "n": 1.5])], text: "Saving."),381 .fail(.timeout),382 ])383 let events = await collect(makeRunner(provider, tools: [writer], approvals: ApprovalRecorder(always: .allowForRun))384 .run(profile: makeProfile(tools: ["write_note"]), goal: "Save a note.", pricing: Pricing(inputPerMTok: 3, outputPerMTok: 15, asOf: "2026-11-01")))385 #expect(events.status == .failed(AstraError.timeout.message))386 #expect(events.checkpoints.map(\.step) == [1, 2]) // one per completed step387 let checkpoint = try #require(events.checkpoints.last)388 let data = try checkpoint.encoded()389 let decoded = try AgentCheckpoint.decode(data)390 #expect(decoded == checkpoint)391 #expect(try decoded.encoded() == data) // stable encoding392 #expect(decoded.version == AgentCheckpoint.currentVersion)393 #expect(decoded.goal == "Save a note." && decoded.profileID == "profile-1")394 #expect(decoded.step == 2 && decoded.toolCalls == 1 && !decoded.isComplete)395 #expect(decoded.messages.count == 5) // goal, 2 assistant turns, 2 tool results396 #expect(decoded.messages[3].toolCalls == [ToolCall(id: "w1", name: "write_note", arguments: ["text": "héllo ✓", "n": 1.5])])397 #expect(decoded.plan?.steps.map(\.title) == ["Write", "Report"])398 #expect(decoded.grants.forRun == ["write_note"])399 #expect(decoded.usage == TokenUsage(inputTokens: 200, outputTokens: 40))400 #expect(decoded.cost.knownTurns == 2 && decoded.cost.isComplete)401 #expect(decoded.recentCalls.count == 1)402 }403404 @Test func newerVersionsAreRefused() async throws {405 let events = await collect(makeRunner(ScriptedProvider([ScriptedProvider.text("hi")])).run(profile: makeProfile(), goal: "g"))406 var checkpoint = try #require(events.checkpoints.last)407 checkpoint.version = AgentCheckpoint.currentVersion + 1408 let data = try checkpoint.encoded()409 #expect(throws: AstraError.self) { try AgentCheckpoint.decode(data) }410 #expect(throws: (any Error).self) { try AgentCheckpoint.decode(Data("{}".utf8)) }411 }412413 @Test func resumesAfterARelaunch() async throws {414 // First launch: one completed step, then the app "dies" (the provider fails on step 2).415 let writer = SpyTool("write_note", risk: .writesUserData)416 let firstProvider = ScriptedProvider([417 ScriptedProvider.calls([ToolCall(id: "p1", name: "update_plan", arguments: ["steps": [["title": "Draft", "status": "in_progress"]]]),418 ToolCall(id: "w1", name: "write_note", arguments: ["text": "draft 1"])]),419 .fail(.network("The connection was lost.")),420 ])421 let firstApprovals = ApprovalRecorder(always: .allowForRun)422 let firstRun = makeRunner(firstProvider, tools: [writer], approvals: firstApprovals).start(profile: makeProfile(tools: ["write_note"]), goal: "Write two notes.")423 let first = await collect(firstRun.events)424 guard case .failed = first.status else { Issue.record("expected the first run to fail"); return }425 let saved = try #require(first.checkpoints.last).encoded() // what the app persisted426427 // Second launch: everything is rebuilt from scratch; only the bytes survive.428 let restored = try AgentCheckpoint.decode(saved)429 let writerAgain = SpyTool("write_note", risk: .writesUserData)430 let secondProvider = ScriptedProvider([ScriptedProvider.call("write_note", ["text": "draft 2"], id: "w2"), ScriptedProvider.text("Both notes are saved.")])431 let secondApprovals = ApprovalRecorder(always: .deny(reason: "should not be asked"))432 let secondRun = makeRunner(secondProvider, tools: [writerAgain], approvals: secondApprovals)433 .start(profile: makeProfile(tools: ["write_note"]), goal: "ignored on resume", resumeFrom: restored)434 let second = await collect(secondRun.events)435436 #expect(secondRun.id == firstRun.id) // same run437 #expect(second.status == .completed && second.finalText == "Both notes are saved.")438 guard case .runStarted(let info) = second.first else { Issue.record("runStarted first"); return }439 #expect(info.resumedAfterStep == 1 && info.goal == "Write two notes." && info.runID == firstRun.id)440 #expect(second.steps == [2, 3]) // counters continue441 #expect(second.plans.first?.steps.map(\.title) == ["Draft"]) // the plan is restored and re-announced442 #expect(secondApprovals.requests.isEmpty) // the "allow for this run" grant survived443 #expect(writerAgain.calls == [["text": "draft 2"]])444 // The model sees the whole history, including the first launch's tool result.445 let history = secondProvider.requests[0].messages446 #expect(history.first == .user("Write two notes."))447 #expect(history.count == 4 && history[2].toolCallID == "p1" && history[3].toolCallID == "w1")448 #expect(history[3].text.contains(UntrustedEnvelope(runID: firstRun.id).closeMarker)) // same envelope nonce across the resume449 let final = try #require(second.checkpoints.last)450 #expect(final.isComplete && final.step == 3 && final.toolCalls == 2)451 #expect(final.usage == TokenUsage(inputTokens: 300, outputTokens: 60)) // usage accumulates across launches452 }453454 @Test func resumeKeepsBudgetsAndCycleHistory() async throws {455 let tool = SpyTool("search")456 let first = await collect(makeRunner(ScriptedProvider([ScriptedProvider.call("search", ["q": "x"]), ScriptedProvider.call("search", ["q": "x"]), .fail(.timeout)]), tools: [tool])457 .run(profile: makeProfile(tools: ["search"]), goal: "g"))458 let checkpoint = try AgentCheckpoint.decode(try #require(first.checkpoints.last).encoded())459 // The third identical call arrives after the relaunch: still a cycle.460 let provider = ScriptedProvider([ScriptedProvider.call("search", ["q": "x"]), ScriptedProvider.text("never")])461 let resumed = await collect(makeRunner(provider, tools: [tool]).run(profile: makeProfile(tools: ["search"]), goal: "", resumeFrom: checkpoint))462 guard case .cycleDetected = resumed.status else { Issue.record("expected a cycle, got \(String(describing: resumed.status))"); return }463 #expect(tool.calls.count == 2)464 // And the step budget counts the steps of the first launch.465 let capped = await collect(makeRunner(ScriptedProvider([ScriptedProvider.text("never")]), tools: [tool])466 .run(profile: makeProfile(tools: ["search"], budgets: AgentBudgets(maxSteps: 2)), goal: "", resumeFrom: checkpoint))467 #expect(capped.status == .budgetExceeded(.steps))468 }469470 @Test func resumeUnderANarrowerProfileDropsToolsAndGrants() async throws {471 let writer = SpyTool("write_note", risk: .writesUserData)472 let first = await collect(makeRunner(ScriptedProvider([ScriptedProvider.call("write_note", ["text": "a"]), .fail(.timeout)]), tools: [writer], approvals: ApprovalRecorder(always: .allowForRun))473 .run(profile: makeProfile(tools: ["write_note"]), goal: "g"))474 var checkpoint = try #require(first.checkpoints.last)475 #expect(checkpoint.grants.forRun == ["write_note"])476 // Even a tampered checkpoint cannot add permissions: the profile decides.477 checkpoint.grants = ApprovalGrants(forRun: ["write_note", "delete_everything", "web_fetch|evil.example"], readsApproved: ["read_file"])478 let provider = ScriptedProvider([ScriptedProvider.call("write_note", ["text": "b"]), ScriptedProvider.text("done")])479 let approvals = ApprovalRecorder(always: .allowOnce)480 let resumed = await collect(makeRunner(provider, tools: [writer, SpyTool("calculator2")], approvals: approvals)481 .run(profile: makeProfile(tools: ["calculator2"]), goal: "", resumeFrom: checkpoint))482 #expect(resumed.status == .completed)483 #expect(writer.calls.count == 1) // only the call from the first launch484 #expect(resumed.finished[0].isError && resumed.finished[0].output.contains("not available to this agent"))485 #expect(resumed.checkpoints.last?.grants == ApprovalGrants())486 #expect(approvals.requests.isEmpty)487 }488489 @Test func resumingAFinishedRunJustReportsItsAnswer() async throws {490 let done = await collect(makeRunner(ScriptedProvider([ScriptedProvider.text("Final answer.")])).run(profile: makeProfile(), goal: "g"))491 let checkpoint = try #require(done.checkpoints.last)492 let provider = ScriptedProvider([ScriptedProvider.text("should not be asked")])493 let again = await collect(makeRunner(provider).run(profile: makeProfile(), goal: "", resumeFrom: checkpoint))494 #expect(again.status == .completed && again.finalText == "Final answer.")495 #expect(provider.requests.isEmpty && again.steps.isEmpty)496 }497}498499// MARK: - Audit500501@Suite struct AgentAuditTests {502 @Test func secretsAreMaskedInEveryAuditEvent() async {503 let api = SpyTool("call_api", risk: .sendsDataExternally) { _ in504 "status: ok\nAuthorization: Bearer abcdefghijklmnopqrstuvwxyz\nsession_token=ZZZZ99998888\nbody: hello sk-ant-api03-SECRETSECRET12345"505 }506 let arguments: JSONValue = [507 "endpoint": "https://api.example.com/v1/items?api_key=QQQQ77776666", "api_key": "sk-proj-abcdefgh12345678", "password": "hunter2!",508 "headers": ["Authorization": "Bearer abcdefghijklmnopqrstuvwxyz", "Accept": "application/json"], "token": "tok_live_1234567890", "query": "public words",509 ]510 let provider = ScriptedProvider([ScriptedProvider.calls([ToolCall(id: "a1", name: "call_api", arguments: arguments)]), ScriptedProvider.text("done")])511 let approvals = ApprovalRecorder(always: .allowOnce)512 let events = await collect(makeRunner(provider, tools: [api], approvals: approvals).run(profile: makeProfile(tools: ["call_api"]), goal: "g"))513 #expect(events.status == .completed)514515 let secrets = ["QQQQ77776666", "abcdefgh12345678", "hunter2", "abcdefghijklmnopqrstuvwxyz", "tok_live_1234567890", "ZZZZ99998888", "SECRETSECRET12345"]516 // Everything the timeline exposes: tool request, approval request, tool output.517 var audit = ""518 for event in events {519 switch event {520 case .toolRequested(let call): audit += call.arguments.serialized()521 case .approvalRequested(let request): audit += request.summary + request.arguments.serialized() + request.warnings.joined()522 case .toolFinished(_, _, let output, _, _): audit += output523 default: break524 }525 }526 for secret in secrets { #expect(!audit.contains(secret), "\(secret) leaked into the audit timeline") }527 #expect(audit.contains("public words") && audit.contains("application/json") && audit.contains("status: ok"))528529 let requested = events.requested[0].arguments530 #expect(requested["api_key"] == "•••" && requested["password"] == "•••" && requested["token"] == "•••")531 #expect(requested["headers"]?["Authorization"] == "•••" && requested["headers"]?["Accept"] == "application/json")532 #expect(requested["query"] == "public words")533 #expect(approvals.requests[0].arguments["password"] == "•••")534 #expect(approvals.requests[0].warnings.contains { $0.contains("key or a password") })535 // The tool itself received the real arguments: redaction is for the audit trail, not for execution.536 #expect(api.calls == [arguments])537 }538539 @Test func longToolOutputIsTruncatedWithAMarker() async {540 let verbose = SpyTool("dump") { _ in String(repeating: "0123456789", count: 5_000) }541 let provider = ScriptedProvider([ScriptedProvider.call("dump", [:], id: "d1"), ScriptedProvider.text("done")])542 let budgets = AgentBudgets(maxToolOutputCharacters: 1_000)543 let events = await collect(makeRunner(provider, tools: [verbose]).run(profile: makeProfile(tools: ["dump"], budgets: budgets), goal: "g"))544 let output = events.finished[0].output545 #expect(output.hasSuffix("[… output truncated: 1000 of 50000 characters shown]"))546 #expect(output.count < 1_100)547 let delivered = toolMessages(provider, request: 1)[0].text548 #expect(delivered.contains("[… output truncated: 1000 of 50000 characters shown]"))549 #expect(delivered.count < 1_600) // the model never receives the 50 000 characters550 #expect(delivered.hasSuffix(">>>")) // and the envelope is still closed551 }552553 @Test func longArgumentsAreShortenedInTheTimeline() async {554 let provider = ScriptedProvider([ScriptedProvider.call("write_file", ["file_name": "big.txt", "content": .string(String(repeating: "word ", count: 20_000))]), ScriptedProvider.text("done")])555 let files = FakeFiles()556 let approvals = ApprovalRecorder(always: .allowOnce)557 let events = await collect(AgentRunner(orchestrator: InferenceOrchestrator(maxRetries: 0) { _ in provider }, registry: .standard(files: files), approvals: approvals)558 .run(profile: makeProfile(tools: ["write_file"]), goal: "g"))559 #expect((events.requested[0].arguments["content"]?.stringValue?.count ?? .max) < 2_100)560 #expect((approvals.requests[0].arguments["content"]?.stringValue?.count ?? .max) < 700)561 #expect(approvals.requests[0].summary.hasPrefix("Save a new file “big.txt” (100000 characters): word word"))562 #expect(files.writes.first?.1.count == 100_000) // the file itself is complete563 }564565 @Test func systemPromptStatesTheRulesButIsNotTheEnforcement() {566 let profile = makeProfile(tools: ["calculator"])567 let runner = makeRunner(ScriptedProvider([]), tools: [CalculatorTool()])568 let prompt = AgentRunner.systemPrompt(profile: profile, tools: runner.offeredTools(for: profile))569 #expect(prompt.hasPrefix("Be exact."))570 #expect(prompt.contains("update_plan, calculator") && prompt.contains("ASTRA_TOOL_RESULT") && prompt.contains("You cannot run code"))571 let bare = AgentRunner.systemPrompt(profile: makeProfile(planning: false), tools: [])572 #expect(bare.contains("You have no tools"))573 }574}575