Repository Public License
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX OS Automation9// File : Packages/ORIXToolRegistry/Tests/ORIXToolRegistryTests/ExecutorPipelineTests.swift10// Description : Unit tests of the registry and of the nine-step11// ToolExecutor pipeline, driven by probe tools and scripted12// policies/brokers.13// Module : ORIXToolRegistryTests14// Repository : https://git.spboucher.ai/orix-automation15// Created : 2026-11-2116// Last Modified : 2026-11-2117//18// =============================================================================19// AUTHOR & PROFESSIONAL AFFILIATION20// =============================================================================21//22// Author : Simon-Pierre Boucher23// Position : Professor of Finance24// Institution : Université du Québec en Outaouais (UQO)25// Department : Département des sciences administratives26// Organization : SP Boucher27//28// Website : https://spboucher.ai29// Git : https://git.spboucher.ai30//31// =============================================================================32// OFFICIAL CONTACT DIRECTORY33// =============================================================================34//35// General : info@spboucher.ai36// Professional : contact@spboucher.ai37// Direct : simon@spboucher.ai38// Research : research@spboucher.ai39// Development : dev@spboucher.ai40// Support : support@spboucher.ai41// Security/Admin : admin@spboucher.ai42// Notifications : notifications@spboucher.ai43//44// Preferred Contact: support@spboucher.ai45//46// =============================================================================47// COPYRIGHT & INTELLECTUAL PROPERTY48// =============================================================================49//50// Copyright (c) 202651// Simon-Pierre Boucher52//53// All rights reserved where applicable.54//55// This file is governed by the applicable project license,56// copyright notices, institutional agreements and relevant law.57//58// Unless expressly authorized by the applicable license or59// rights holder, copying, redistribution, sublicensing,60// publication and commercial exploitation are not permitted.61//62// Existing third-party rights and notices remain unaffected.63//64// =============================================================================65// CONFIDENTIALITY & SECURITY66// =============================================================================67//68// Classification : Public69// Distribution : Public repository; use is governed by the project license70// Security Contact : admin@spboucher.ai71//72// Access to this source does not independently grant permission73// to redistribute or disclose confidential materials.74//75// Do not embed credentials, private keys, access tokens,76// certificates or confidential personal information.77//78// =============================================================================79// LICENSE & MAINTENANCE80// =============================================================================81//82// License : Proprietary, all rights reserved (see license file)83// License File : LICENSE84// Maintainer : Simon-Pierre Boucher85// Technical Contact: dev@spboucher.ai86//87// =============================================================================88// END OF SP BOUCHER SOURCE HEADER89// =============================================================================9091import Foundation92import Testing93import ORIXCore94import ORIXTestingSupport95@testable import ORIXToolRegistry9697@Suite("Manifests and registry (unit)")98struct ManifestAndRegistryTests {99 @Test func everyBuiltInManifestIsValid() {100 for tool in ORIXToolRegistryTools.all() {101 #expect(tool.manifest.problems().isEmpty, "\(tool.identifier): \(tool.manifest.problems())")102 #expect(tool.identifier == tool.manifest.identifier)103 #expect(tool.manifest.family == String(tool.identifier.split(separator: ".")[0]))104 #expect(tool.manifest.outputSchema.raw["type"]?.stringValue == "object", "\(tool.identifier) must declare an object output")105 }106 }107108 @Test func identifiersMatchTheCanonicalList() {109 let ids = ORIXToolRegistryTools.all().map(\.identifier)110 #expect(Set(ids).count == ids.count)111 #expect(Set(ids) == Set(ORIXToolRegistryTools.canonicalIdentifiers))112 #expect(ids.count == 44)113 }114115 @Test func tiersFollowTheGuide() {116 let byID = Dictionary(uniqueKeysWithValues: ORIXToolRegistryTools.all().map { ($0.identifier, $0.manifest) })117 #expect(byID["filesystem.delete"]?.defaultTier == .highRisk)118 #expect(byID["filesystem.delete"]?.sideEffect == .destructive)119 #expect(byID["filesystem.trash"]?.defaultTier == .reversibleWrite)120 #expect(byID["git.commit"]?.defaultTier == .reversibleWrite)121 for id in ["shell.run", "shell.script", "python.run", "script.swift", "script.javascript", "git.push", "http.request"] {122 #expect(byID[id]?.defaultTier == .consequential, "\(id)")123 }124 for (id, m) in byID where id.hasPrefix("system.") { #expect(m.defaultTier == .readOnly && m.sideEffect == .read) }125 #expect(!byID.keys.contains { $0.contains("reset") || $0.contains("clean") || $0.contains("rebase") })126 }127128 @Test func registryRejectsDuplicatesAndBadManifests() async throws {129 let registry = ToolRegistry()130 try await registry.register(FilesystemReadTool())131 await #expect(throws: ToolRegistryError.duplicateIdentifier("filesystem.read")) { try await registry.register(FilesystemReadTool()) }132 var bad = ProbeTool("test.bad") { _, _ in .success([:], summary: "") }133 bad.manifest.summary = "short"134 await #expect(throws: ToolRegistryError.self) { try await registry.register(bad) }135 // registerAll is atomic.136 await #expect(throws: ToolRegistryError.self) { try await registry.registerAll([FilesystemListTool(), bad]) }137 #expect(await registry.tool(id: "filesystem.list") == nil)138 try await registry.registerAll([FilesystemListTool(), SystemInfoTool()])139 #expect(await registry.manifests().map(\.identifier) == ["filesystem.list", "filesystem.read", "system.info"])140 #expect(await registry.manifests(family: "filesystem").count == 2)141 #expect(await registry.unregister(id: "system.info"))142 #expect(await registry.unregister(id: "system.info") == false)143 #expect(await registry.tool(id: "system.info") == nil)144 }145}146147@Suite("ToolExecutor pipeline (unit, probe tools)")148struct ExecutorPipelineTests {149 static func echo(_ counter: Counter = Counter()) -> ProbeTool {150 ProbeTool("test.echo") { input, _ in counter.increment(); return .success(["text": .string(input.optionalString("text") ?? "")], summary: "echo") }151 }152 static func slow(_ counter: Counter = Counter(), id: String = "test.slow", concurrency: ToolConcurrency = .unlimited) -> ProbeTool {153 ProbeTool(id, concurrency: concurrency) { input, _ in154 counter.enter(); defer { counter.leave() }155 try await Task.sleep(for: .seconds(input.double("seconds", default: 0.15)))156 return .success([:], summary: "slept")157 }158 }159 static func writer(_ counter: Counter, id: String = "test.write") -> ProbeTool {160 ProbeTool(id, sideEffect: .write, tier: .reversibleWrite) { _, _ in counter.increment(); return .success(["n": .int(counter.value)], summary: "wrote") }161 }162163 @Test func unknownToolIsNotFound() async throws {164 let h = try await Harness(builtIns: false)165 defer { h.tmp.remove() }166 let r = await h.call("nope.missing")167 #expect(r.status == .failed && r.failure?.code == "not_found")168 #expect(await h.events.events(of: .toolFinished).count == 1)169 #expect(await h.audit.entries.last?.kind == .toolFailed)170 }171172 @Test func schemaViolationRunsNothing() async throws {173 let counter = Counter()174 let h = try await Harness(extra: [Self.echo(counter)], builtIns: false)175 defer { h.tmp.remove() }176 let r = await h.call("test.echo", ["text": 42])177 #expect(r.status == .invalidInput && r.failure?.code == "invalid_input")178 let extra = await h.call("test.echo", ["unexpected": true])179 #expect(extra.status == .invalidInput)180 #expect(counter.value == 0)181 }182183 @Test func protectedPathIsDeniedAndAuditedAsViolation() async throws {184 let h = try await Harness(policy: AllowAllPolicy())185 defer { h.tmp.remove() }186 let r = await h.call("filesystem.read", ["path": "~/.ssh/id_ed25519"])187 #expect(r.status == .denied && r.failure?.code == "permission_denied")188 #expect(await h.audit.entries.contains { $0.kind == .policyViolation && $0.toolID == "filesystem.read" })189 let w = await h.call("filesystem.write", ["path": "~/Library/Keychains/x", "content": "x"])190 #expect(w.status == .denied)191 #expect(!FileManager.default.fileExists(atPath: NSHomeDirectory() + "/Library/Keychains/x"))192 }193194 @Test func denyAllPolicyDenies() async throws {195 let counter = Counter()196 let h = try await Harness(policy: DenyAllPolicy(), extra: [Self.echo(counter)], builtIns: false)197 defer { h.tmp.remove() }198 let r = await h.call("test.echo", ["text": "a"])199 #expect(r.status == .denied && counter.value == 0)200 #expect(await h.audit.entries.last?.kind == .toolDenied)201 }202203 @Test func approvalOnceExecutesAndConsumesTheGrant() async throws {204 let counter = Counter()205 let policy = ApprovalPolicy()206 let broker = ScriptedApprovalBroker(answers: [.approveOnce])207 let h = try await Harness(policy: policy, approvals: broker, extra: [Self.writer(counter)], builtIns: false)208 defer { h.tmp.remove() }209 let r = await h.call("test.write", [:], id: "call-1")210 #expect(r.status == .succeeded && counter.value == 1)211 #expect(await broker.requests.count == 1)212 #expect(await policy.approvals == [.approveOnce])213 #expect(await policy.consumed == ["grant-call-1"])214 #expect(await h.events.events(of: .approvalRequested).count == 1)215 #expect(await h.events.events(of: .approvalResolved).first?.data["approved"] == .bool(true))216 let kinds = await h.audit.entries.map(\.kind)217 #expect(kinds.contains(.approvalRequested) && kinds.contains(.approvalGranted) && kinds.last == .toolExecuted)218 // The scripted list is exhausted: the next call falls back to deny.219 let again = await h.call("test.write")220 #expect(again.status == .notApproved && counter.value == 1)221 }222223 @Test func refusedOrUnansweredApprovalIsNotApproved() async throws {224 let counter = Counter()225 let broker = ScriptedApprovalBroker(answers: [.deny(reason: "not now"), .unanswered])226 let h = try await Harness(policy: ApprovalPolicy(), approvals: broker, extra: [Self.writer(counter)], builtIns: false)227 defer { h.tmp.remove() }228 for _ in 0..<2 {229 let r = await h.call("test.write")230 #expect(r.status == .notApproved && r.failure?.code == "not_approved")231 }232 #expect(counter.value == 0)233 #expect(await h.audit.entries.filter { $0.kind == .approvalDenied }.count == 2)234 #expect(await h.audit.entries.last?.kind == .toolDenied)235 }236237 @Test func approvalWaitDoesNotBlockOtherCalls() async throws {238 let counter = Counter(), broker = GatedBroker()239 let h = try await Harness(policy: ApprovalPolicy(), approvals: broker, extra: [Self.writer(counter), Self.echo()], builtIns: false)240 defer { h.tmp.remove() }241 async let pending = h.call("test.write")242 while await broker.asked == 0 { try await Task.sleep(for: .milliseconds(10)) }243 let other = await h.call("test.echo", ["text": "meanwhile"], runID: "run-other")244 #expect(other.status == .succeeded)245 #expect(await h.executor.inFlightCount == 1)246 await broker.release(.approveOnce)247 #expect(await pending.status == .succeeded)248 #expect(await h.executor.inFlightCount == 0)249 }250251 @Test func idempotencyKeyReplaysWithoutExecuting() async throws {252 let counter = Counter()253 let h = try await Harness(policy: AllowAllPolicy(), extra: [Self.writer(counter)], builtIns: false)254 defer { h.tmp.remove() }255 let first = await h.call("test.write", [:], key: "k1")256 let second = await h.call("test.write", [:], key: "k1")257 #expect(first.output == second.output && counter.value == 1)258 #expect(await h.events.events(of: .toolFinished).last?.data["replayed"] == .bool(true))259 _ = await h.call("test.write", [:], key: "k1", runID: "another-run")260 #expect(counter.value == 2)261 _ = await h.call("test.write")262 _ = await h.call("test.write")263 #expect(counter.value == 4)264 }265266 @Test func concurrentCallsWithTheSameKeyExecuteOnce() async throws {267 let counter = Counter()268 let tool = ProbeTool("test.slowwrite", sideEffect: .write, tier: .reversibleWrite) { _, _ in269 counter.increment(); try await Task.sleep(for: .milliseconds(150)); return .success(["n": 1], summary: "ok")270 }271 let h = try await Harness(policy: AllowAllPolicy(), extra: [tool], builtIns: false)272 defer { h.tmp.remove() }273 async let a = h.call("test.slowwrite", [:], key: "same")274 async let b = h.call("test.slowwrite", [:], key: "same")275 let (ra, rb) = await (a, b)276 #expect(ra.status == .succeeded && rb.status == .succeeded && counter.value == 1)277 }278279 @Test func failedCallsAreNotStoredForReplay() async throws {280 let counter = Counter()281 let tool = ProbeTool("test.flaky", sideEffect: .write, tier: .reversibleWrite) { _, _ in282 counter.increment()283 if counter.value == 1 { throw ToolError.io("disk hiccup") }284 return .success([:], summary: "ok")285 }286 let h = try await Harness(policy: AllowAllPolicy(), extra: [tool], builtIns: false)287 defer { h.tmp.remove() }288 #expect(await h.call("test.flaky", [:], key: "k").status == .failed)289 #expect(await h.call("test.flaky", [:], key: "k").status == .succeeded)290 #expect(counter.value == 2)291 }292293 @Test func timeoutFromArgumentAndFromRunBound() async throws {294 let h = try await Harness(policy: AllowAllPolicy(), extra: [Self.slow()], builtIns: false)295 defer { h.tmp.remove() }296 let clock = ContinuousClock()297 let started = clock.now298 let r = await h.call("test.slow", ["seconds": 20, "timeoutSeconds": 0.3])299 #expect(r.status == .timedOut && r.failure?.code == "timeout")300 let bounded = await h.call("test.slow", ["seconds": 20], maxToolSeconds: 0.3)301 #expect(bounded.status == .timedOut)302 #expect(clock.now - started < .seconds(5))303 #expect(await h.audit.entries.last?.kind == .toolFailed)304 }305306 @Test func cancelAllAndCancelRun() async throws {307 let h = try await Harness(policy: AllowAllPolicy(), extra: [Self.slow()], builtIns: false)308 defer { h.tmp.remove() }309 async let a = h.call("test.slow", ["seconds": 30], runID: "run-a")310 async let b = h.call("test.slow", ["seconds": 30], runID: "run-b")311 while await h.executor.inFlightCount < 2 { try await Task.sleep(for: .milliseconds(10)) }312 try await Task.sleep(for: .milliseconds(50))313 await h.executor.cancel(runID: "run-a")314 #expect(await a.status == .canceled)315 #expect(await h.executor.inFlightCount == 1)316 await h.executor.cancelAll()317 let rb = await b318 #expect(rb.status == .canceled && rb.failure?.code == "canceled")319 #expect(await h.executor.inFlightCount == 0)320 }321322 @Test func cancellingTheCallingTaskCancelsTheTool() async throws {323 let h = try await Harness(policy: AllowAllPolicy(), extra: [Self.slow()], builtIns: false)324 defer { h.tmp.remove() }325 let task = Task { await h.call("test.slow", ["seconds": 30]) }326 while await h.executor.inFlightCount < 1 { try await Task.sleep(for: .milliseconds(10)) }327 try await Task.sleep(for: .milliseconds(50))328 task.cancel()329 #expect(await task.value.status == .canceled)330 }331332 @Test func outputAndEventsAreRedacted() async throws {333 let secret = "correct-horse-battery-staple-9431"334 let shaped = "sk-ant-api03-" + String(repeating: "Ab3xY9", count: 8)335 let tool = ProbeTool("test.leak", redactedKeys: ["apiKey"]) { _, ctx in336 ctx.progress(ToolProgress(stream: .stdout, text: "progress \(secret)"))337 return .success(["line": .string("value=\(secret) and \(shaped)"), "nested": ["items": [.string(shaped)]]], summary: "leaked \(secret)")338 }339 let h = try await Harness(policy: AllowAllPolicy(), extra: [tool], builtIns: false)340 defer { h.tmp.remove() }341 h.redactor.register(secret: secret)342 let seen = Counter()343 var context = h.tmp.invocation()344 final class Box: @unchecked Sendable { var text = "" }345 let box = Box()346 context.progress = { _, p in box.text += p.text; seen.increment() }347 let r = await h.executor.invoke(ToolCall(toolID: "test.leak", arguments: ["apiKey": "hunter2-hunter2", "text": .string("see \(shaped)")]), context: context)348 let whole = r.output.serialized() + r.summary349 #expect(r.status == .succeeded)350 #expect(!whole.contains(secret) && !whole.contains(shaped) && whole.contains(Redactor.placeholder))351 #expect(seen.value == 1 && !box.text.contains(secret))352 let events = await h.events.events353 let dump = events.map { $0.text + $0.data.serialized() }.joined() + (await h.audit.entries.map { $0.detail + $0.data.serialized() }.joined())354 #expect(!dump.contains(secret) && !dump.contains(shaped) && !dump.contains("hunter2-hunter2"))355 #expect(events.first?.data["arguments"]?["apiKey"] == .string(Redactor.placeholder))356 }357358 @Test func invalidOutputAndThrownErrorsAreTyped() async throws {359 let schema = JSONSchema.object(["n": .integer()], required: ["n"], additional: true)360 let badOutput = ProbeTool("test.badout", outputSchema: schema) { _, _ in .success(["n": "not a number"], summary: "x") }361 let tcc = ProbeTool("test.tcc") { _, _ in throw ToolError.tccBlocked(.accessibility, "Accessibility is off") }362 let unsupported = ProbeTool("test.unsupported") { _, _ in throw ToolError.unsupported("not on this Mac") }363 struct Odd: Error {}364 let odd = ProbeTool("test.odd") { _, _ in throw Odd() }365 let h = try await Harness(policy: AllowAllPolicy(), extra: [badOutput, tcc, unsupported, odd], builtIns: false)366 defer { h.tmp.remove() }367 let r1 = await h.call("test.badout")368 #expect(r1.status == .failed && r1.failure?.code == "output_invalid")369 let r2 = await h.call("test.tcc")370 #expect(r2.status == .blockedPermission && r2.failure?.code == "tcc_blocked")371 #expect(await h.call("test.unsupported").status == .unsupported)372 let r4 = await h.call("test.odd")373 #expect(r4.status == .failed && r4.failure?.code == "internal")374 }375376 @Test func oversizedOutputIsTruncated() async throws {377 let tool = ProbeTool("test.big", maxOutputBytes: 10_000) { _, _ in .success(["text": .string(String(repeating: "x", count: 500_000)), "n": 7], summary: "big") }378 let h = try await Harness(policy: AllowAllPolicy(), extra: [tool], builtIns: false)379 defer { h.tmp.remove() }380 let r = await h.call("test.big")381 #expect(r.status == .succeeded && r.truncated)382 #expect(r.output.serialized().utf8.count <= 10_000)383 #expect(r.output["n"] == 7)384 let (preview, cut) = ToolExecutor.bounded(.array((0..<5_000).map { .string("item-\($0)") }), maxBytes: 2_000)385 #expect(cut && preview["truncated"] == true && preview.serialized().utf8.count <= 2_000)386 }387388 @Test func eventsAndAuditDescribeTheCall() async throws {389 let h = try await Harness(policy: AllowAllPolicy(), extra: [Self.echo(), ProbeTool("test.web", untrusted: true) { _, _ in .success([:], summary: "page") }], builtIns: false)390 defer { h.tmp.remove() }391 let r = await h.call("test.echo", ["text": "hi"], id: "call-42")392 #expect(r.durationMs >= 0 && !r.untrusted)393 let requested = await h.events.events(of: .toolRequested), finished = await h.events.events(of: .toolFinished)394 #expect(requested.count == 1 && finished.count == 1)395 #expect(requested[0].data["callID"] == "call-42" && requested[0].data["toolID"] == "test.echo")396 let data = finished[0].data397 #expect(data["status"] == "succeeded" && data["tier"] == 0 && data["summary"] == "echo" && data["durationMs"]?.intValue != nil)398 #expect(data["arguments"]?["text"] == "hi")399 let entry = await h.audit.entries.last400 #expect(entry?.kind == .toolExecuted && entry?.toolID == "test.echo" && entry?.runID == "run-test" && entry?.tier == .readOnly)401 #expect(await h.call("test.web").untrusted)402 }403404 @Test func serialToolsRunOneAtATime() async throws {405 let serial = Counter(), free = Counter()406 let h = try await Harness(policy: AllowAllPolicy(),407 extra: [Self.slow(serial, id: "test.serial", concurrency: .serial), Self.slow(free, id: "test.free")], builtIns: false)408 defer { h.tmp.remove() }409 await withTaskGroup(of: ToolResult.self) { group in410 for i in 0..<5 {411 group.addTask { await h.call("test.serial", ["seconds": 0.05], runID: "run-\(i)") }412 group.addTask { await h.call("test.free", ["seconds": 0.2], runID: "run-\(i)") }413 }414 for await r in group { #expect(r.status == .succeeded) }415 }416 #expect(serial.value == 5 && serial.maxConcurrent == 1)417 #expect(free.value == 5 && free.maxConcurrent > 1)418 }419420 @Test func exclusiveToolsExcludeEachOther() async throws {421 let shared = Counter()422 let h = try await Harness(policy: AllowAllPolicy(), extra: [Self.slow(shared, id: "test.pointer", concurrency: .exclusive),423 Self.slow(shared, id: "test.keyboard", concurrency: .exclusive)], builtIns: false)424 defer { h.tmp.remove() }425 await withTaskGroup(of: ToolResult.self) { group in426 for i in 0..<6 { group.addTask { await h.call(i % 2 == 0 ? "test.pointer" : "test.keyboard", ["seconds": 0.04]) } }427 for await r in group { #expect(r.status == .succeeded) }428 }429 #expect(shared.value == 6 && shared.maxConcurrent == 1)430 }431432 @Test func toolIgnoringCancellationIsAbandonedAfterTheGracePeriod() async throws {433 let finished = Counter()434 let stubborn = ProbeTool("test.stubborn") { _, _ in435 // Deliberately not cancellable.436 await withCheckedContinuation { (c: CheckedContinuation<Void, Never>) in DispatchQueue.global().asyncAfter(deadline: .now() + 8) { c.resume() } }437 finished.increment()438 return .success([:], summary: "late")439 }440 let h = try await Harness(policy: AllowAllPolicy(), extra: [stubborn], builtIns: false)441 defer { h.tmp.remove() }442 await h.executor.setUnwindGrace(0.3)443 let clock = ContinuousClock(); let start = clock.now444 let r = await h.call("test.stubborn", ["timeoutSeconds": 0.2])445 #expect(r.status == .timedOut && finished.value == 0 && clock.now - start < .seconds(5), "the caller is answered even though the tool is still running")446 let task = Task { await h.call("test.stubborn") }447 while await h.executor.inFlightCount < 1 { try await Task.sleep(for: .milliseconds(10)) }448 try await Task.sleep(for: .milliseconds(50))449 await h.executor.cancelAll()450 #expect(await task.value.status == .canceled && finished.value == 0 && clock.now - start < .seconds(7))451 }452453 @Test func dryRunIsRefusedWhenUnsupported() async throws {454 let counter = Counter()455 let h = try await Harness(policy: AllowAllPolicy(), extra: [Self.writer(counter)], builtIns: false)456 defer { h.tmp.remove() }457 let r = await h.call("test.write", [:], dryRun: true)458 #expect(r.status == .unsupported && counter.value == 0)459 }460461 @Test func artifactsDirectoryIsImplicitlyGranted() async throws {462 let policy = ApprovalPolicy()463 let h = try await Harness(policy: policy, approvals: ScriptedApprovalBroker(answers: [.approveOnce, .approveOnce]))464 defer { h.tmp.remove() }465 let inside = await h.call("filesystem.write", ["path": "artifacts/note.txt", "content": "x"])466 #expect(inside.status == .succeeded)467 #expect(await policy.requests.last?.capabilities.isEmpty == true)468 let outside = await h.call("filesystem.write", ["path": "note.txt", "content": "x"])469 #expect(outside.status == .succeeded)470 #expect(await policy.requests.last?.capabilities == [.fileWrite(path: h.path("note.txt"))])471 #expect(await policy.requests.last?.summary.contains("note.txt") == true)472 }473474 @Test func secretsAreScopedToDeclaredReferences() async throws {475 var declared = ProbeTool("test.secret") { _, ctx in476 let v = try await ctx.secrets?.secret(reference: "api-token") ?? "none"477 return .success(["echo": .string("the value is \(v)")], summary: "used")478 }479 declared.capabilities = { _, _ in [.secret(reference: "api-token")] }480 let sneaky = ProbeTool("test.sneaky") { _, ctx in481 _ = try await ctx.secrets?.secret(reference: "api-token")482 return .success([:], summary: "stole")483 }484 let h = try await Harness(policy: AllowAllPolicy(), extra: [declared, sneaky], secrets: MapSecrets(values: ["api-token": "tok-7Qm2-very-secret-value"]), builtIns: false)485 defer { h.tmp.remove() }486 let ok = await h.call("test.secret")487 #expect(ok.status == .succeeded && !ok.output.serialized().contains("tok-7Qm2") && ok.output.serialized().contains(Redactor.placeholder))488 #expect(await h.audit.entries.contains { $0.kind == .secretUsed && $0.data["reference"] == "api-token" })489 #expect(await h.call("test.sneaky").status == .denied)490 }491492 @Test func requirementErrorsStopBeforePolicy() async throws {493 let policy = ApprovalPolicy()494 let h = try await Harness(policy: policy)495 defer { h.tmp.remove() }496 let r = await h.call("filesystem.read", ["path": "~root/secret"])497 #expect(r.status == .invalidInput)498 let sudo = await h.call("shell.run", ["executable": "sudo", "args": ["ls"]])499 #expect(sudo.status == .unsupported)500 #expect(await policy.requests.isEmpty)501 }502}503