SPB Git

Repository Public License

TestingSupport.swift 8.8 KB 180 lines swift

1// ORIX OS Automation2// File: Packages/ORIXCore/Sources/ORIXTestingSupport/TestingSupport.swift3// Description: Test doubles shared by every package: scripted provider, fixed policies, scripted approval broker, collecting sinks, temp directories, closure-backed tool invoker.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import ORIXCore910/// A provider that replays scripted turns. Clearly a mock: its id is `scripted` unless stated otherwise.11public final class ScriptedProvider: LLMProvider, @unchecked Sendable {12    public enum Turn: Sendable {13        case response(LLMResponse)14        case failure(LLMError)15        /// Computes the turn from the request (to assert on what the runtime sent).16        case dynamic(@Sendable (LLMRequest) -> LLMResponse)17    }18    public let id: ProviderID19    private let lock = NSLock()20    private var turns: [Turn]21    private var _requests: [LLMRequest] = []22    /// When the script runs out: repeat this turn forever (used for loop/budget tests), else fail.23    public var fallback: Turn?2425    public init(id: ProviderID = ProviderID(rawValue: "scripted"), turns: [Turn], fallback: Turn? = nil) {26        self.id = id; self.turns = turns; self.fallback = fallback27    }2829    public var requests: [LLMRequest] { lock.lock(); defer { lock.unlock() }; return _requests }3031    public func listModels() async throws -> [ModelInfo] {32        [ModelInfo(id: "scripted-1", provider: id, inputPricePerMTok: 1, outputPricePerMTok: 2, source: "registry")]33    }3435    public func stream(_ request: LLMRequest) -> AsyncThrowingStream<LLMStreamEvent, Error> {36        lock.lock()37        _requests.append(request)38        let turn: Turn? = turns.isEmpty ? fallback : turns.removeFirst()39        lock.unlock()40        return AsyncThrowingStream { continuation in41            guard let turn else {42                continuation.finish(throwing: LLMError.invalidRequest("ScriptedProvider has no more turns")); return43            }44            let response: LLMResponse45            switch turn {46            case .failure(let e): continuation.finish(throwing: e); return47            case .response(let r): response = r48            case .dynamic(let f): response = f(request)49            }50            for c in response.content {51                switch c {52                case .text(let t): continuation.yield(.textDelta(t))53                case .toolUse(let u): continuation.yield(.toolUse(u))54                default: break55                }56            }57            continuation.yield(.usage(response.usage))58            continuation.yield(.stop(response.stopReason))59            continuation.finish()60        }61    }6263    public static func text(_ text: String, usage: TokenUsage = .init(inputTokens: 10, outputTokens: 5)) -> Turn {64        .response(LLMResponse(model: "scripted-1", content: [.text(text)], stopReason: .endTurn, usage: usage))65    }6667    public static func tool(_ toolID: String, _ input: JSONValue, id: String = UUID().uuidString, text: String = "",68                            usage: TokenUsage = .init(inputTokens: 10, outputTokens: 5)) -> Turn {69        var content: [LLMContent] = text.isEmpty ? [] : [.text(text)]70        content.append(.toolUse(LLMToolUse(id: id, name: LLMToolSpec.providerName(forToolID: toolID), input: input)))71        return .response(LLMResponse(model: "scripted-1", content: content, stopReason: .toolUse, usage: usage))72    }73}7475public struct StaticProviderResolver: ProviderResolving {76    public let provider: any LLMProvider77    public init(_ provider: any LLMProvider) { self.provider = provider }78    public func provider(for id: ProviderID) async throws -> any LLMProvider { provider }79    public func modelInfo(_ ref: ModelRef) async -> ModelInfo? {80        ModelInfo(id: ref.model, provider: ref.provider, inputPricePerMTok: 1, outputPricePerMTok: 2, source: "registry")81    }82}8384public struct AllowAllPolicy: PolicyDeciding {85    public init() {}86    public func evaluate(_ request: AccessRequest) async -> PolicyDecision { .allow(grantIDs: []) }87    public func recordApproval(_ response: ApprovalResponse, for request: ApprovalRequest) async -> [String] { [] }88    public func consume(grantIDs: [String], callID: String) async {}89    public func confinement(runID: String, workflowID: String?) async -> Confinement { Confinement(readRoots: ["/"], writeRoots: ["/"], networkHosts: ["*"]) }90}9192public struct DenyAllPolicy: PolicyDeciding {93    public init() {}94    public func evaluate(_ request: AccessRequest) async -> PolicyDecision { .deny(reason: "DenyAllPolicy") }95    public func recordApproval(_ response: ApprovalResponse, for request: ApprovalRequest) async -> [String] { [] }96    public func consume(grantIDs: [String], callID: String) async {}97    public func confinement(runID: String, workflowID: String?) async -> Confinement { Confinement() }98}99100/// Answers approval requests from a fixed list, then with `fallback`. Records what it was asked.101public actor ScriptedApprovalBroker: ApprovalBroker {102    private var answers: [ApprovalResponse]103    private let fallback: ApprovalResponse104    public private(set) var requests: [ApprovalRequest] = []105    public init(answers: [ApprovalResponse] = [], fallback: ApprovalResponse = .deny(reason: "no scripted answer")) {106        self.answers = answers; self.fallback = fallback107    }108    public func requestApproval(_ request: ApprovalRequest) async -> ApprovalResponse {109        requests.append(request)110        return answers.isEmpty ? fallback : answers.removeFirst()111    }112}113114public actor CollectingEventSink: RunEventSink {115    public private(set) var events: [RunEvent] = []116    public init() {}117    public func emit(_ event: RunEvent) async {118        var e = event; e.seq = events.count + 1119        events.append(e)120    }121    public func events(of kind: RunEvent.Kind) -> [RunEvent] { events.filter { $0.kind == kind } }122}123124public actor CollectingAuditSink: AuditSink {125    public private(set) var entries: [AuditEntry] = []126    public init() {}127    public func record(_ entry: AuditEntry) async { entries.append(entry) }128}129130/// A tool invoker backed by closures, for agent and workflow tests that do not need the real executor.131public actor ClosureToolInvoker: ToolInvoking {132    public typealias Handler = @Sendable (ToolCall, InvocationContext) async -> ToolResult133    private var handlers: [String: Handler] = [:]134    private var specs: [ToolManifest] = []135    public private(set) var calls: [ToolCall] = []136    public init() {}137138    public func register(_ id: String, summary: String = "Test tool for unit tests.", schema: JSONSchema = .object([:], additional: true),139                         sideEffect: SideEffect = .read, tier: ApprovalTier = .readOnly, idempotent: Bool = true, handler: @escaping Handler) {140        handlers[id] = handler141        specs.append(ToolManifest(identifier: id, name: id, summary: summary, family: String(id.split(separator: ".").first ?? "test"),142                                  inputSchema: schema, sideEffect: sideEffect, defaultTier: tier, idempotent: idempotent))143    }144145    public func manifests() async -> [ToolManifest] { specs }146147    public func invoke(_ call: ToolCall, context: InvocationContext) async -> ToolResult {148        calls.append(call)149        guard let h = handlers[call.toolID] else {150            return .failure(.failed, ToolFailure(code: "not_found", message: "Unknown tool \(call.toolID)"))151        }152        return await h(call, context)153    }154}155156/// A unique temporary directory removed on `remove()`.157public struct TempDirectory: Sendable {158    public let url: URL159    public init(prefix: String = "orix-test") throws {160        let base = URL(fileURLWithPath: PathResolver.resolveSymlinks(NSTemporaryDirectory()))161        url = base.appendingPathComponent("\(prefix)-\(UUID().uuidString.prefix(8))")162        try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true)163    }164    public func remove() { try? FileManager.default.removeItem(at: url) }165    public func file(_ name: String) -> URL { url.appendingPathComponent(name) }166    @discardableResult public func write(_ name: String, _ text: String) throws -> URL {167        let f = file(name)168        try FileManager.default.createDirectory(at: f.deletingLastPathComponent(), withIntermediateDirectories: true)169        try text.write(to: f, atomically: true, encoding: .utf8)170        return f171    }172    public func invocation(runID: String = "run-test", mode: ExecutionMode = .guided) -> InvocationContext {173        InvocationContext(runID: runID, mode: mode, workingDirectory: url, artifactsDirectory: url.appendingPathComponent("artifacts"))174    }175    public func toolContext(runID: String = "run-test") -> ToolExecutionContext {176        ToolExecutionContext(runID: runID, workingDirectory: url, artifactsDirectory: url.appendingPathComponent("artifacts"),177                             confinement: Confinement(readRoots: [url.path], writeRoots: [url.path]))178    }179}180
Files main
Menu

Theme