SPB Git

Repository Public License

ExecutorPipelineTests.swift 26.8 KB 503 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX OS Automation9// File             : Packages/ORIXToolRegistry/Tests/ORIXToolRegistryTests/ExecutorPipelineTests.swift10// Description      : Unit tests of the registry and of the nine-step11//                    ToolExecutor pipeline, driven by probe tools and scripted12//                    policies/brokers.13// Module           : ORIXToolRegistryTests14// Repository       : https://git.spboucher.ai/orix-automation15// Created          : 2026-11-2116// Last Modified    : 2026-11-2117//18// =============================================================================19// AUTHOR & PROFESSIONAL AFFILIATION20// =============================================================================21//22// Author           : Simon-Pierre Boucher23// Position         : Professor of Finance24// Institution      : Université du Québec en Outaouais (UQO)25// Department       : Département des sciences administratives26// Organization     : SP Boucher27//28// Website          : https://spboucher.ai29// Git              : https://git.spboucher.ai30//31// =============================================================================32// OFFICIAL CONTACT DIRECTORY33// =============================================================================34//35// General          : info@spboucher.ai36// Professional     : contact@spboucher.ai37// Direct           : simon@spboucher.ai38// Research         : research@spboucher.ai39// Development      : dev@spboucher.ai40// Support          : support@spboucher.ai41// Security/Admin   : admin@spboucher.ai42// Notifications    : notifications@spboucher.ai43//44// Preferred Contact: support@spboucher.ai45//46// =============================================================================47// COPYRIGHT & INTELLECTUAL PROPERTY48// =============================================================================49//50// Copyright (c) 202651// Simon-Pierre Boucher52//53// All rights reserved where applicable.54//55// This file is governed by the applicable project license,56// copyright notices, institutional agreements and relevant law.57//58// Unless expressly authorized by the applicable license or59// rights holder, copying, redistribution, sublicensing,60// publication and commercial exploitation are not permitted.61//62// Existing third-party rights and notices remain unaffected.63//64// =============================================================================65// CONFIDENTIALITY & SECURITY66// =============================================================================67//68// Classification   : Public69// Distribution     : Public repository; use is governed by the project license70// Security Contact : admin@spboucher.ai71//72// Access to this source does not independently grant permission73// to redistribute or disclose confidential materials.74//75// Do not embed credentials, private keys, access tokens,76// certificates or confidential personal information.77//78// =============================================================================79// LICENSE & MAINTENANCE80// =============================================================================81//82// License          : Proprietary, all rights reserved (see license file)83// License File     : LICENSE84// Maintainer       : Simon-Pierre Boucher85// Technical Contact: dev@spboucher.ai86//87// =============================================================================88// END OF SP BOUCHER SOURCE HEADER89// =============================================================================9091import Foundation92import Testing93import ORIXCore94import ORIXTestingSupport95@testable import ORIXToolRegistry9697@Suite("Manifests and registry (unit)")98struct ManifestAndRegistryTests {99    @Test func everyBuiltInManifestIsValid() {100        for tool in ORIXToolRegistryTools.all() {101            #expect(tool.manifest.problems().isEmpty, "\(tool.identifier): \(tool.manifest.problems())")102            #expect(tool.identifier == tool.manifest.identifier)103            #expect(tool.manifest.family == String(tool.identifier.split(separator: ".")[0]))104            #expect(tool.manifest.outputSchema.raw["type"]?.stringValue == "object", "\(tool.identifier) must declare an object output")105        }106    }107108    @Test func identifiersMatchTheCanonicalList() {109        let ids = ORIXToolRegistryTools.all().map(\.identifier)110        #expect(Set(ids).count == ids.count)111        #expect(Set(ids) == Set(ORIXToolRegistryTools.canonicalIdentifiers))112        #expect(ids.count == 44)113    }114115    @Test func tiersFollowTheGuide() {116        let byID = Dictionary(uniqueKeysWithValues: ORIXToolRegistryTools.all().map { ($0.identifier, $0.manifest) })117        #expect(byID["filesystem.delete"]?.defaultTier == .highRisk)118        #expect(byID["filesystem.delete"]?.sideEffect == .destructive)119        #expect(byID["filesystem.trash"]?.defaultTier == .reversibleWrite)120        #expect(byID["git.commit"]?.defaultTier == .reversibleWrite)121        for id in ["shell.run", "shell.script", "python.run", "script.swift", "script.javascript", "git.push", "http.request"] {122            #expect(byID[id]?.defaultTier == .consequential, "\(id)")123        }124        for (id, m) in byID where id.hasPrefix("system.") { #expect(m.defaultTier == .readOnly && m.sideEffect == .read) }125        #expect(!byID.keys.contains { $0.contains("reset") || $0.contains("clean") || $0.contains("rebase") })126    }127128    @Test func registryRejectsDuplicatesAndBadManifests() async throws {129        let registry = ToolRegistry()130        try await registry.register(FilesystemReadTool())131        await #expect(throws: ToolRegistryError.duplicateIdentifier("filesystem.read")) { try await registry.register(FilesystemReadTool()) }132        var bad = ProbeTool("test.bad") { _, _ in .success([:], summary: "") }133        bad.manifest.summary = "short"134        await #expect(throws: ToolRegistryError.self) { try await registry.register(bad) }135        // registerAll is atomic.136        await #expect(throws: ToolRegistryError.self) { try await registry.registerAll([FilesystemListTool(), bad]) }137        #expect(await registry.tool(id: "filesystem.list") == nil)138        try await registry.registerAll([FilesystemListTool(), SystemInfoTool()])139        #expect(await registry.manifests().map(\.identifier) == ["filesystem.list", "filesystem.read", "system.info"])140        #expect(await registry.manifests(family: "filesystem").count == 2)141        #expect(await registry.unregister(id: "system.info"))142        #expect(await registry.unregister(id: "system.info") == false)143        #expect(await registry.tool(id: "system.info") == nil)144    }145}146147@Suite("ToolExecutor pipeline (unit, probe tools)")148struct ExecutorPipelineTests {149    static func echo(_ counter: Counter = Counter()) -> ProbeTool {150        ProbeTool("test.echo") { input, _ in counter.increment(); return .success(["text": .string(input.optionalString("text") ?? "")], summary: "echo") }151    }152    static func slow(_ counter: Counter = Counter(), id: String = "test.slow", concurrency: ToolConcurrency = .unlimited) -> ProbeTool {153        ProbeTool(id, concurrency: concurrency) { input, _ in154            counter.enter(); defer { counter.leave() }155            try await Task.sleep(for: .seconds(input.double("seconds", default: 0.15)))156            return .success([:], summary: "slept")157        }158    }159    static func writer(_ counter: Counter, id: String = "test.write") -> ProbeTool {160        ProbeTool(id, sideEffect: .write, tier: .reversibleWrite) { _, _ in counter.increment(); return .success(["n": .int(counter.value)], summary: "wrote") }161    }162163    @Test func unknownToolIsNotFound() async throws {164        let h = try await Harness(builtIns: false)165        defer { h.tmp.remove() }166        let r = await h.call("nope.missing")167        #expect(r.status == .failed && r.failure?.code == "not_found")168        #expect(await h.events.events(of: .toolFinished).count == 1)169        #expect(await h.audit.entries.last?.kind == .toolFailed)170    }171172    @Test func schemaViolationRunsNothing() async throws {173        let counter = Counter()174        let h = try await Harness(extra: [Self.echo(counter)], builtIns: false)175        defer { h.tmp.remove() }176        let r = await h.call("test.echo", ["text": 42])177        #expect(r.status == .invalidInput && r.failure?.code == "invalid_input")178        let extra = await h.call("test.echo", ["unexpected": true])179        #expect(extra.status == .invalidInput)180        #expect(counter.value == 0)181    }182183    @Test func protectedPathIsDeniedAndAuditedAsViolation() async throws {184        let h = try await Harness(policy: AllowAllPolicy())185        defer { h.tmp.remove() }186        let r = await h.call("filesystem.read", ["path": "~/.ssh/id_ed25519"])187        #expect(r.status == .denied && r.failure?.code == "permission_denied")188        #expect(await h.audit.entries.contains { $0.kind == .policyViolation && $0.toolID == "filesystem.read" })189        let w = await h.call("filesystem.write", ["path": "~/Library/Keychains/x", "content": "x"])190        #expect(w.status == .denied)191        #expect(!FileManager.default.fileExists(atPath: NSHomeDirectory() + "/Library/Keychains/x"))192    }193194    @Test func denyAllPolicyDenies() async throws {195        let counter = Counter()196        let h = try await Harness(policy: DenyAllPolicy(), extra: [Self.echo(counter)], builtIns: false)197        defer { h.tmp.remove() }198        let r = await h.call("test.echo", ["text": "a"])199        #expect(r.status == .denied && counter.value == 0)200        #expect(await h.audit.entries.last?.kind == .toolDenied)201    }202203    @Test func approvalOnceExecutesAndConsumesTheGrant() async throws {204        let counter = Counter()205        let policy = ApprovalPolicy()206        let broker = ScriptedApprovalBroker(answers: [.approveOnce])207        let h = try await Harness(policy: policy, approvals: broker, extra: [Self.writer(counter)], builtIns: false)208        defer { h.tmp.remove() }209        let r = await h.call("test.write", [:], id: "call-1")210        #expect(r.status == .succeeded && counter.value == 1)211        #expect(await broker.requests.count == 1)212        #expect(await policy.approvals == [.approveOnce])213        #expect(await policy.consumed == ["grant-call-1"])214        #expect(await h.events.events(of: .approvalRequested).count == 1)215        #expect(await h.events.events(of: .approvalResolved).first?.data["approved"] == .bool(true))216        let kinds = await h.audit.entries.map(\.kind)217        #expect(kinds.contains(.approvalRequested) && kinds.contains(.approvalGranted) && kinds.last == .toolExecuted)218        // The scripted list is exhausted: the next call falls back to deny.219        let again = await h.call("test.write")220        #expect(again.status == .notApproved && counter.value == 1)221    }222223    @Test func refusedOrUnansweredApprovalIsNotApproved() async throws {224        let counter = Counter()225        let broker = ScriptedApprovalBroker(answers: [.deny(reason: "not now"), .unanswered])226        let h = try await Harness(policy: ApprovalPolicy(), approvals: broker, extra: [Self.writer(counter)], builtIns: false)227        defer { h.tmp.remove() }228        for _ in 0..<2 {229            let r = await h.call("test.write")230            #expect(r.status == .notApproved && r.failure?.code == "not_approved")231        }232        #expect(counter.value == 0)233        #expect(await h.audit.entries.filter { $0.kind == .approvalDenied }.count == 2)234        #expect(await h.audit.entries.last?.kind == .toolDenied)235    }236237    @Test func approvalWaitDoesNotBlockOtherCalls() async throws {238        let counter = Counter(), broker = GatedBroker()239        let h = try await Harness(policy: ApprovalPolicy(), approvals: broker, extra: [Self.writer(counter), Self.echo()], builtIns: false)240        defer { h.tmp.remove() }241        async let pending = h.call("test.write")242        while await broker.asked == 0 { try await Task.sleep(for: .milliseconds(10)) }243        let other = await h.call("test.echo", ["text": "meanwhile"], runID: "run-other")244        #expect(other.status == .succeeded)245        #expect(await h.executor.inFlightCount == 1)246        await broker.release(.approveOnce)247        #expect(await pending.status == .succeeded)248        #expect(await h.executor.inFlightCount == 0)249    }250251    @Test func idempotencyKeyReplaysWithoutExecuting() async throws {252        let counter = Counter()253        let h = try await Harness(policy: AllowAllPolicy(), extra: [Self.writer(counter)], builtIns: false)254        defer { h.tmp.remove() }255        let first = await h.call("test.write", [:], key: "k1")256        let second = await h.call("test.write", [:], key: "k1")257        #expect(first.output == second.output && counter.value == 1)258        #expect(await h.events.events(of: .toolFinished).last?.data["replayed"] == .bool(true))259        _ = await h.call("test.write", [:], key: "k1", runID: "another-run")260        #expect(counter.value == 2)261        _ = await h.call("test.write")262        _ = await h.call("test.write")263        #expect(counter.value == 4)264    }265266    @Test func concurrentCallsWithTheSameKeyExecuteOnce() async throws {267        let counter = Counter()268        let tool = ProbeTool("test.slowwrite", sideEffect: .write, tier: .reversibleWrite) { _, _ in269            counter.increment(); try await Task.sleep(for: .milliseconds(150)); return .success(["n": 1], summary: "ok")270        }271        let h = try await Harness(policy: AllowAllPolicy(), extra: [tool], builtIns: false)272        defer { h.tmp.remove() }273        async let a = h.call("test.slowwrite", [:], key: "same")274        async let b = h.call("test.slowwrite", [:], key: "same")275        let (ra, rb) = await (a, b)276        #expect(ra.status == .succeeded && rb.status == .succeeded && counter.value == 1)277    }278279    @Test func failedCallsAreNotStoredForReplay() async throws {280        let counter = Counter()281        let tool = ProbeTool("test.flaky", sideEffect: .write, tier: .reversibleWrite) { _, _ in282            counter.increment()283            if counter.value == 1 { throw ToolError.io("disk hiccup") }284            return .success([:], summary: "ok")285        }286        let h = try await Harness(policy: AllowAllPolicy(), extra: [tool], builtIns: false)287        defer { h.tmp.remove() }288        #expect(await h.call("test.flaky", [:], key: "k").status == .failed)289        #expect(await h.call("test.flaky", [:], key: "k").status == .succeeded)290        #expect(counter.value == 2)291    }292293    @Test func timeoutFromArgumentAndFromRunBound() async throws {294        let h = try await Harness(policy: AllowAllPolicy(), extra: [Self.slow()], builtIns: false)295        defer { h.tmp.remove() }296        let clock = ContinuousClock()297        let started = clock.now298        let r = await h.call("test.slow", ["seconds": 20, "timeoutSeconds": 0.3])299        #expect(r.status == .timedOut && r.failure?.code == "timeout")300        let bounded = await h.call("test.slow", ["seconds": 20], maxToolSeconds: 0.3)301        #expect(bounded.status == .timedOut)302        #expect(clock.now - started < .seconds(5))303        #expect(await h.audit.entries.last?.kind == .toolFailed)304    }305306    @Test func cancelAllAndCancelRun() async throws {307        let h = try await Harness(policy: AllowAllPolicy(), extra: [Self.slow()], builtIns: false)308        defer { h.tmp.remove() }309        async let a = h.call("test.slow", ["seconds": 30], runID: "run-a")310        async let b = h.call("test.slow", ["seconds": 30], runID: "run-b")311        while await h.executor.inFlightCount < 2 { try await Task.sleep(for: .milliseconds(10)) }312        try await Task.sleep(for: .milliseconds(50))313        await h.executor.cancel(runID: "run-a")314        #expect(await a.status == .canceled)315        #expect(await h.executor.inFlightCount == 1)316        await h.executor.cancelAll()317        let rb = await b318        #expect(rb.status == .canceled && rb.failure?.code == "canceled")319        #expect(await h.executor.inFlightCount == 0)320    }321322    @Test func cancellingTheCallingTaskCancelsTheTool() async throws {323        let h = try await Harness(policy: AllowAllPolicy(), extra: [Self.slow()], builtIns: false)324        defer { h.tmp.remove() }325        let task = Task { await h.call("test.slow", ["seconds": 30]) }326        while await h.executor.inFlightCount < 1 { try await Task.sleep(for: .milliseconds(10)) }327        try await Task.sleep(for: .milliseconds(50))328        task.cancel()329        #expect(await task.value.status == .canceled)330    }331332    @Test func outputAndEventsAreRedacted() async throws {333        let secret = "correct-horse-battery-staple-9431"334        let shaped = "sk-ant-api03-" + String(repeating: "Ab3xY9", count: 8)335        let tool = ProbeTool("test.leak", redactedKeys: ["apiKey"]) { _, ctx in336            ctx.progress(ToolProgress(stream: .stdout, text: "progress \(secret)"))337            return .success(["line": .string("value=\(secret) and \(shaped)"), "nested": ["items": [.string(shaped)]]], summary: "leaked \(secret)")338        }339        let h = try await Harness(policy: AllowAllPolicy(), extra: [tool], builtIns: false)340        defer { h.tmp.remove() }341        h.redactor.register(secret: secret)342        let seen = Counter()343        var context = h.tmp.invocation()344        final class Box: @unchecked Sendable { var text = "" }345        let box = Box()346        context.progress = { _, p in box.text += p.text; seen.increment() }347        let r = await h.executor.invoke(ToolCall(toolID: "test.leak", arguments: ["apiKey": "hunter2-hunter2", "text": .string("see \(shaped)")]), context: context)348        let whole = r.output.serialized() + r.summary349        #expect(r.status == .succeeded)350        #expect(!whole.contains(secret) && !whole.contains(shaped) && whole.contains(Redactor.placeholder))351        #expect(seen.value == 1 && !box.text.contains(secret))352        let events = await h.events.events353        let dump = events.map { $0.text + $0.data.serialized() }.joined() + (await h.audit.entries.map { $0.detail + $0.data.serialized() }.joined())354        #expect(!dump.contains(secret) && !dump.contains(shaped) && !dump.contains("hunter2-hunter2"))355        #expect(events.first?.data["arguments"]?["apiKey"] == .string(Redactor.placeholder))356    }357358    @Test func invalidOutputAndThrownErrorsAreTyped() async throws {359        let schema = JSONSchema.object(["n": .integer()], required: ["n"], additional: true)360        let badOutput = ProbeTool("test.badout", outputSchema: schema) { _, _ in .success(["n": "not a number"], summary: "x") }361        let tcc = ProbeTool("test.tcc") { _, _ in throw ToolError.tccBlocked(.accessibility, "Accessibility is off") }362        let unsupported = ProbeTool("test.unsupported") { _, _ in throw ToolError.unsupported("not on this Mac") }363        struct Odd: Error {}364        let odd = ProbeTool("test.odd") { _, _ in throw Odd() }365        let h = try await Harness(policy: AllowAllPolicy(), extra: [badOutput, tcc, unsupported, odd], builtIns: false)366        defer { h.tmp.remove() }367        let r1 = await h.call("test.badout")368        #expect(r1.status == .failed && r1.failure?.code == "output_invalid")369        let r2 = await h.call("test.tcc")370        #expect(r2.status == .blockedPermission && r2.failure?.code == "tcc_blocked")371        #expect(await h.call("test.unsupported").status == .unsupported)372        let r4 = await h.call("test.odd")373        #expect(r4.status == .failed && r4.failure?.code == "internal")374    }375376    @Test func oversizedOutputIsTruncated() async throws {377        let tool = ProbeTool("test.big", maxOutputBytes: 10_000) { _, _ in .success(["text": .string(String(repeating: "x", count: 500_000)), "n": 7], summary: "big") }378        let h = try await Harness(policy: AllowAllPolicy(), extra: [tool], builtIns: false)379        defer { h.tmp.remove() }380        let r = await h.call("test.big")381        #expect(r.status == .succeeded && r.truncated)382        #expect(r.output.serialized().utf8.count <= 10_000)383        #expect(r.output["n"] == 7)384        let (preview, cut) = ToolExecutor.bounded(.array((0..<5_000).map { .string("item-\($0)") }), maxBytes: 2_000)385        #expect(cut && preview["truncated"] == true && preview.serialized().utf8.count <= 2_000)386    }387388    @Test func eventsAndAuditDescribeTheCall() async throws {389        let h = try await Harness(policy: AllowAllPolicy(), extra: [Self.echo(), ProbeTool("test.web", untrusted: true) { _, _ in .success([:], summary: "page") }], builtIns: false)390        defer { h.tmp.remove() }391        let r = await h.call("test.echo", ["text": "hi"], id: "call-42")392        #expect(r.durationMs >= 0 && !r.untrusted)393        let requested = await h.events.events(of: .toolRequested), finished = await h.events.events(of: .toolFinished)394        #expect(requested.count == 1 && finished.count == 1)395        #expect(requested[0].data["callID"] == "call-42" && requested[0].data["toolID"] == "test.echo")396        let data = finished[0].data397        #expect(data["status"] == "succeeded" && data["tier"] == 0 && data["summary"] == "echo" && data["durationMs"]?.intValue != nil)398        #expect(data["arguments"]?["text"] == "hi")399        let entry = await h.audit.entries.last400        #expect(entry?.kind == .toolExecuted && entry?.toolID == "test.echo" && entry?.runID == "run-test" && entry?.tier == .readOnly)401        #expect(await h.call("test.web").untrusted)402    }403404    @Test func serialToolsRunOneAtATime() async throws {405        let serial = Counter(), free = Counter()406        let h = try await Harness(policy: AllowAllPolicy(),407                                  extra: [Self.slow(serial, id: "test.serial", concurrency: .serial), Self.slow(free, id: "test.free")], builtIns: false)408        defer { h.tmp.remove() }409        await withTaskGroup(of: ToolResult.self) { group in410            for i in 0..<5 {411                group.addTask { await h.call("test.serial", ["seconds": 0.05], runID: "run-\(i)") }412                group.addTask { await h.call("test.free", ["seconds": 0.2], runID: "run-\(i)") }413            }414            for await r in group { #expect(r.status == .succeeded) }415        }416        #expect(serial.value == 5 && serial.maxConcurrent == 1)417        #expect(free.value == 5 && free.maxConcurrent > 1)418    }419420    @Test func exclusiveToolsExcludeEachOther() async throws {421        let shared = Counter()422        let h = try await Harness(policy: AllowAllPolicy(), extra: [Self.slow(shared, id: "test.pointer", concurrency: .exclusive),423                                                                    Self.slow(shared, id: "test.keyboard", concurrency: .exclusive)], builtIns: false)424        defer { h.tmp.remove() }425        await withTaskGroup(of: ToolResult.self) { group in426            for i in 0..<6 { group.addTask { await h.call(i % 2 == 0 ? "test.pointer" : "test.keyboard", ["seconds": 0.04]) } }427            for await r in group { #expect(r.status == .succeeded) }428        }429        #expect(shared.value == 6 && shared.maxConcurrent == 1)430    }431432    @Test func toolIgnoringCancellationIsAbandonedAfterTheGracePeriod() async throws {433        let finished = Counter()434        let stubborn = ProbeTool("test.stubborn") { _, _ in435            // Deliberately not cancellable.436            await withCheckedContinuation { (c: CheckedContinuation<Void, Never>) in DispatchQueue.global().asyncAfter(deadline: .now() + 8) { c.resume() } }437            finished.increment()438            return .success([:], summary: "late")439        }440        let h = try await Harness(policy: AllowAllPolicy(), extra: [stubborn], builtIns: false)441        defer { h.tmp.remove() }442        await h.executor.setUnwindGrace(0.3)443        let clock = ContinuousClock(); let start = clock.now444        let r = await h.call("test.stubborn", ["timeoutSeconds": 0.2])445        #expect(r.status == .timedOut && finished.value == 0 && clock.now - start < .seconds(5), "the caller is answered even though the tool is still running")446        let task = Task { await h.call("test.stubborn") }447        while await h.executor.inFlightCount < 1 { try await Task.sleep(for: .milliseconds(10)) }448        try await Task.sleep(for: .milliseconds(50))449        await h.executor.cancelAll()450        #expect(await task.value.status == .canceled && finished.value == 0 && clock.now - start < .seconds(7))451    }452453    @Test func dryRunIsRefusedWhenUnsupported() async throws {454        let counter = Counter()455        let h = try await Harness(policy: AllowAllPolicy(), extra: [Self.writer(counter)], builtIns: false)456        defer { h.tmp.remove() }457        let r = await h.call("test.write", [:], dryRun: true)458        #expect(r.status == .unsupported && counter.value == 0)459    }460461    @Test func artifactsDirectoryIsImplicitlyGranted() async throws {462        let policy = ApprovalPolicy()463        let h = try await Harness(policy: policy, approvals: ScriptedApprovalBroker(answers: [.approveOnce, .approveOnce]))464        defer { h.tmp.remove() }465        let inside = await h.call("filesystem.write", ["path": "artifacts/note.txt", "content": "x"])466        #expect(inside.status == .succeeded)467        #expect(await policy.requests.last?.capabilities.isEmpty == true)468        let outside = await h.call("filesystem.write", ["path": "note.txt", "content": "x"])469        #expect(outside.status == .succeeded)470        #expect(await policy.requests.last?.capabilities == [.fileWrite(path: h.path("note.txt"))])471        #expect(await policy.requests.last?.summary.contains("note.txt") == true)472    }473474    @Test func secretsAreScopedToDeclaredReferences() async throws {475        var declared = ProbeTool("test.secret") { _, ctx in476            let v = try await ctx.secrets?.secret(reference: "api-token") ?? "none"477            return .success(["echo": .string("the value is \(v)")], summary: "used")478        }479        declared.capabilities = { _, _ in [.secret(reference: "api-token")] }480        let sneaky = ProbeTool("test.sneaky") { _, ctx in481            _ = try await ctx.secrets?.secret(reference: "api-token")482            return .success([:], summary: "stole")483        }484        let h = try await Harness(policy: AllowAllPolicy(), extra: [declared, sneaky], secrets: MapSecrets(values: ["api-token": "tok-7Qm2-very-secret-value"]), builtIns: false)485        defer { h.tmp.remove() }486        let ok = await h.call("test.secret")487        #expect(ok.status == .succeeded && !ok.output.serialized().contains("tok-7Qm2") && ok.output.serialized().contains(Redactor.placeholder))488        #expect(await h.audit.entries.contains { $0.kind == .secretUsed && $0.data["reference"] == "api-token" })489        #expect(await h.call("test.sneaky").status == .denied)490    }491492    @Test func requirementErrorsStopBeforePolicy() async throws {493        let policy = ApprovalPolicy()494        let h = try await Harness(policy: policy)495        defer { h.tmp.remove() }496        let r = await h.call("filesystem.read", ["path": "~root/secret"])497        #expect(r.status == .invalidInput)498        let sudo = await h.call("shell.run", ["executable": "sudo", "args": ["ls"]])499        #expect(sudo.status == .unsupported)500        #expect(await policy.requests.isEmpty)501    }502}503
Files main
Menu

Theme