Repository Public License
1// ORIX OS Automation2// File: Packages/ORIXCore/Sources/ORIXTestingSupport/TestingSupport.swift3// Description: Test doubles shared by every package: scripted provider, fixed policies, scripted approval broker, collecting sinks, temp directories, closure-backed tool invoker.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import ORIXCore910/// A provider that replays scripted turns. Clearly a mock: its id is `scripted` unless stated otherwise.11public final class ScriptedProvider: LLMProvider, @unchecked Sendable {12 public enum Turn: Sendable {13 case response(LLMResponse)14 case failure(LLMError)15 /// Computes the turn from the request (to assert on what the runtime sent).16 case dynamic(@Sendable (LLMRequest) -> LLMResponse)17 }18 public let id: ProviderID19 private let lock = NSLock()20 private var turns: [Turn]21 private var _requests: [LLMRequest] = []22 /// When the script runs out: repeat this turn forever (used for loop/budget tests), else fail.23 public var fallback: Turn?2425 public init(id: ProviderID = ProviderID(rawValue: "scripted"), turns: [Turn], fallback: Turn? = nil) {26 self.id = id; self.turns = turns; self.fallback = fallback27 }2829 public var requests: [LLMRequest] { lock.lock(); defer { lock.unlock() }; return _requests }3031 public func listModels() async throws -> [ModelInfo] {32 [ModelInfo(id: "scripted-1", provider: id, inputPricePerMTok: 1, outputPricePerMTok: 2, source: "registry")]33 }3435 public func stream(_ request: LLMRequest) -> AsyncThrowingStream<LLMStreamEvent, Error> {36 lock.lock()37 _requests.append(request)38 let turn: Turn? = turns.isEmpty ? fallback : turns.removeFirst()39 lock.unlock()40 return AsyncThrowingStream { continuation in41 guard let turn else {42 continuation.finish(throwing: LLMError.invalidRequest("ScriptedProvider has no more turns")); return43 }44 let response: LLMResponse45 switch turn {46 case .failure(let e): continuation.finish(throwing: e); return47 case .response(let r): response = r48 case .dynamic(let f): response = f(request)49 }50 for c in response.content {51 switch c {52 case .text(let t): continuation.yield(.textDelta(t))53 case .toolUse(let u): continuation.yield(.toolUse(u))54 default: break55 }56 }57 continuation.yield(.usage(response.usage))58 continuation.yield(.stop(response.stopReason))59 continuation.finish()60 }61 }6263 public static func text(_ text: String, usage: TokenUsage = .init(inputTokens: 10, outputTokens: 5)) -> Turn {64 .response(LLMResponse(model: "scripted-1", content: [.text(text)], stopReason: .endTurn, usage: usage))65 }6667 public static func tool(_ toolID: String, _ input: JSONValue, id: String = UUID().uuidString, text: String = "",68 usage: TokenUsage = .init(inputTokens: 10, outputTokens: 5)) -> Turn {69 var content: [LLMContent] = text.isEmpty ? [] : [.text(text)]70 content.append(.toolUse(LLMToolUse(id: id, name: LLMToolSpec.providerName(forToolID: toolID), input: input)))71 return .response(LLMResponse(model: "scripted-1", content: content, stopReason: .toolUse, usage: usage))72 }73}7475public struct StaticProviderResolver: ProviderResolving {76 public let provider: any LLMProvider77 public init(_ provider: any LLMProvider) { self.provider = provider }78 public func provider(for id: ProviderID) async throws -> any LLMProvider { provider }79 public func modelInfo(_ ref: ModelRef) async -> ModelInfo? {80 ModelInfo(id: ref.model, provider: ref.provider, inputPricePerMTok: 1, outputPricePerMTok: 2, source: "registry")81 }82}8384public struct AllowAllPolicy: PolicyDeciding {85 public init() {}86 public func evaluate(_ request: AccessRequest) async -> PolicyDecision { .allow(grantIDs: []) }87 public func recordApproval(_ response: ApprovalResponse, for request: ApprovalRequest) async -> [String] { [] }88 public func consume(grantIDs: [String], callID: String) async {}89 public func confinement(runID: String, workflowID: String?) async -> Confinement { Confinement(readRoots: ["/"], writeRoots: ["/"], networkHosts: ["*"]) }90}9192public struct DenyAllPolicy: PolicyDeciding {93 public init() {}94 public func evaluate(_ request: AccessRequest) async -> PolicyDecision { .deny(reason: "DenyAllPolicy") }95 public func recordApproval(_ response: ApprovalResponse, for request: ApprovalRequest) async -> [String] { [] }96 public func consume(grantIDs: [String], callID: String) async {}97 public func confinement(runID: String, workflowID: String?) async -> Confinement { Confinement() }98}99100/// Answers approval requests from a fixed list, then with `fallback`. Records what it was asked.101public actor ScriptedApprovalBroker: ApprovalBroker {102 private var answers: [ApprovalResponse]103 private let fallback: ApprovalResponse104 public private(set) var requests: [ApprovalRequest] = []105 public init(answers: [ApprovalResponse] = [], fallback: ApprovalResponse = .deny(reason: "no scripted answer")) {106 self.answers = answers; self.fallback = fallback107 }108 public func requestApproval(_ request: ApprovalRequest) async -> ApprovalResponse {109 requests.append(request)110 return answers.isEmpty ? fallback : answers.removeFirst()111 }112}113114public actor CollectingEventSink: RunEventSink {115 public private(set) var events: [RunEvent] = []116 public init() {}117 public func emit(_ event: RunEvent) async {118 var e = event; e.seq = events.count + 1119 events.append(e)120 }121 public func events(of kind: RunEvent.Kind) -> [RunEvent] { events.filter { $0.kind == kind } }122}123124public actor CollectingAuditSink: AuditSink {125 public private(set) var entries: [AuditEntry] = []126 public init() {}127 public func record(_ entry: AuditEntry) async { entries.append(entry) }128}129130/// A tool invoker backed by closures, for agent and workflow tests that do not need the real executor.131public actor ClosureToolInvoker: ToolInvoking {132 public typealias Handler = @Sendable (ToolCall, InvocationContext) async -> ToolResult133 private var handlers: [String: Handler] = [:]134 private var specs: [ToolManifest] = []135 public private(set) var calls: [ToolCall] = []136 public init() {}137138 public func register(_ id: String, summary: String = "Test tool for unit tests.", schema: JSONSchema = .object([:], additional: true),139 sideEffect: SideEffect = .read, tier: ApprovalTier = .readOnly, idempotent: Bool = true, handler: @escaping Handler) {140 handlers[id] = handler141 specs.append(ToolManifest(identifier: id, name: id, summary: summary, family: String(id.split(separator: ".").first ?? "test"),142 inputSchema: schema, sideEffect: sideEffect, defaultTier: tier, idempotent: idempotent))143 }144145 public func manifests() async -> [ToolManifest] { specs }146147 public func invoke(_ call: ToolCall, context: InvocationContext) async -> ToolResult {148 calls.append(call)149 guard let h = handlers[call.toolID] else {150 return .failure(.failed, ToolFailure(code: "not_found", message: "Unknown tool \(call.toolID)"))151 }152 return await h(call, context)153 }154}155156/// A unique temporary directory removed on `remove()`.157public struct TempDirectory: Sendable {158 public let url: URL159 public init(prefix: String = "orix-test") throws {160 let base = URL(fileURLWithPath: PathResolver.resolveSymlinks(NSTemporaryDirectory()))161 url = base.appendingPathComponent("\(prefix)-\(UUID().uuidString.prefix(8))")162 try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true)163 }164 public func remove() { try? FileManager.default.removeItem(at: url) }165 public func file(_ name: String) -> URL { url.appendingPathComponent(name) }166 @discardableResult public func write(_ name: String, _ text: String) throws -> URL {167 let f = file(name)168 try FileManager.default.createDirectory(at: f.deletingLastPathComponent(), withIntermediateDirectories: true)169 try text.write(to: f, atomically: true, encoding: .utf8)170 return f171 }172 public func invocation(runID: String = "run-test", mode: ExecutionMode = .guided) -> InvocationContext {173 InvocationContext(runID: runID, mode: mode, workingDirectory: url, artifactsDirectory: url.appendingPathComponent("artifacts"))174 }175 public func toolContext(runID: String = "run-test") -> ToolExecutionContext {176 ToolExecutionContext(runID: runID, workingDirectory: url, artifactsDirectory: url.appendingPathComponent("artifacts"),177 confinement: Confinement(readRoots: [url.path], writeRoots: [url.path]))178 }179}180