SPB Git

Repository Public License

TestSupport.swift 13.1 KB 259 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX OS Automation9// File             : Packages/ORIXToolRegistry/Tests/ORIXToolRegistryTests/TestSupport.swift10// Description      : Shared test fixtures: an executor harness over a temp11//                    directory, configurable policies, probe tools and process12//                    helpers.13// Module           : ORIXToolRegistryTests14// Repository       : https://git.spboucher.ai/orix-automation15// Created          : 2026-11-2116// Last Modified    : 2026-11-2117//18// =============================================================================19// AUTHOR & PROFESSIONAL AFFILIATION20// =============================================================================21//22// Author           : Simon-Pierre Boucher23// Position         : Professor of Finance24// Institution      : Université du Québec en Outaouais (UQO)25// Department       : Département des sciences administratives26// Organization     : SP Boucher27//28// Website          : https://spboucher.ai29// Git              : https://git.spboucher.ai30//31// =============================================================================32// OFFICIAL CONTACT DIRECTORY33// =============================================================================34//35// General          : info@spboucher.ai36// Professional     : contact@spboucher.ai37// Direct           : simon@spboucher.ai38// Research         : research@spboucher.ai39// Development      : dev@spboucher.ai40// Support          : support@spboucher.ai41// Security/Admin   : admin@spboucher.ai42// Notifications    : notifications@spboucher.ai43//44// Preferred Contact: support@spboucher.ai45//46// =============================================================================47// COPYRIGHT & INTELLECTUAL PROPERTY48// =============================================================================49//50// Copyright (c) 202651// Simon-Pierre Boucher52//53// All rights reserved where applicable.54//55// This file is governed by the applicable project license,56// copyright notices, institutional agreements and relevant law.57//58// Unless expressly authorized by the applicable license or59// rights holder, copying, redistribution, sublicensing,60// publication and commercial exploitation are not permitted.61//62// Existing third-party rights and notices remain unaffected.63//64// =============================================================================65// CONFIDENTIALITY & SECURITY66// =============================================================================67//68// Classification   : Public69// Distribution     : Public repository; use is governed by the project license70// Security Contact : admin@spboucher.ai71//72// Access to this source does not independently grant permission73// to redistribute or disclose confidential materials.74//75// Do not embed credentials, private keys, access tokens,76// certificates or confidential personal information.77//78// =============================================================================79// LICENSE & MAINTENANCE80// =============================================================================81//82// License          : Proprietary, all rights reserved (see license file)83// License File     : LICENSE84// Maintainer       : Simon-Pierre Boucher85// Technical Contact: dev@spboucher.ai86//87// =============================================================================88// END OF SP BOUCHER SOURCE HEADER89// =============================================================================9091import Foundation92import ORIXCore93import ORIXTestingSupport94@testable import ORIXToolRegistry9596/// Allows everything but confines subprocesses to the given envelope.97struct ConfinedAllowPolicy: PolicyDeciding {98    let confinement: Confinement99    func evaluate(_ request: AccessRequest) async -> PolicyDecision { .allow(grantIDs: []) }100    func recordApproval(_ response: ApprovalResponse, for request: ApprovalRequest) async -> [String] { [] }101    func consume(grantIDs: [String], callID: String) async {}102    func confinement(runID: String, workflowID: String?) async -> Confinement { confinement }103}104105/// Small custom policy: tier 0 is allowed, everything else needs approval once per call; records what it saw.106actor ApprovalPolicy: PolicyDeciding {107    private(set) var requests: [AccessRequest] = []108    private(set) var approvals: [ApprovalResponse] = []109    private(set) var consumed: [String] = []110    private var runGrants: Set<String> = []111    let confinementValue: Confinement112    let denyTools: Set<String>113114    init(confinement: Confinement = Confinement(), denyTools: Set<String> = []) { confinementValue = confinement; self.denyTools = denyTools }115116    func evaluate(_ request: AccessRequest) async -> PolicyDecision {117        requests.append(request)118        if denyTools.contains(request.toolID) { return .deny(reason: "tool is on the deny list") }119        if request.tier == .readOnly || runGrants.contains(request.runID + request.toolID) { return .allow(grantIDs: []) }120        return .needsApproval(ApprovalRequest(access: request, missing: request.capabilities, reason: "no grant covers this call"))121    }122    func recordApproval(_ response: ApprovalResponse, for request: ApprovalRequest) async -> [String] {123        approvals.append(response)124        if response == .approveForRun { runGrants.insert(request.access.runID + request.access.toolID) }125        return ["grant-\(request.access.callID)"]126    }127    func consume(grantIDs: [String], callID: String) async { consumed += grantIDs }128    func confinement(runID: String, workflowID: String?) async -> Confinement { confinementValue }129}130131/// A broker whose answer is held until `release` is called.132actor GatedBroker: ApprovalBroker {133    private var waiting: [CheckedContinuation<ApprovalResponse, Never>] = []134    private(set) var asked = 0135    func requestApproval(_ request: ApprovalRequest) async -> ApprovalResponse {136        asked += 1137        return await withCheckedContinuation { waiting.append($0) }138    }139    func release(_ response: ApprovalResponse) { waiting.forEach { $0.resume(returning: response) }; waiting = [] }140}141142struct MapSecrets: SecretResolving {143    let values: [String: String]144    func secret(reference: String) async throws -> String {145        guard let v = values[reference] else { throw ToolError.notFound("no secret \(reference)") }146        return v147    }148}149150final class Counter: @unchecked Sendable {151    private let lock = NSLock()152    private var n = 0, live = 0, peak = 0153    var value: Int { lock.lock(); defer { lock.unlock() }; return n }154    var maxConcurrent: Int { lock.lock(); defer { lock.unlock() }; return peak }155    func increment() { lock.lock(); n += 1; lock.unlock() }156    func enter() { lock.lock(); live += 1; peak = max(peak, live); n += 1; lock.unlock() }157    func leave() { lock.lock(); live -= 1; lock.unlock() }158}159160/// A configurable tool for pipeline tests. Clearly a test double: family `test`.161struct ProbeTool: ORIXTool {162    var manifest: ToolManifest163    var capabilities: @Sendable (ToolInput, ToolExecutionContext) throws -> [Capability] = { _, _ in [] }164    var tier: ApprovalTier?165    var body: @Sendable (ToolInput, ToolExecutionContext) async throws -> ToolResult166167    init(_ id: String, sideEffect: SideEffect = .read, tier: ApprovalTier = .readOnly, concurrency: ToolConcurrency = .unlimited,168         idempotent: Bool = false, supportsDryRun: Bool = false, maxOutputBytes: Int = 262_144, timeout: Double = 30,169         redactedKeys: [String] = [], untrusted: Bool = false,170         outputSchema: JSONSchema = .any,171         body: @escaping @Sendable (ToolInput, ToolExecutionContext) async throws -> ToolResult) {172        manifest = ToolManifest(identifier: id, name: id, summary: "Probe tool used by the executor tests.", family: "test",173                                inputSchema: .object(["text": .string(), "seconds": .number(), "timeoutSeconds": .number(), "apiKey": .string(), "path": .string()], additional: false),174                                outputSchema: outputSchema, sideEffect: sideEffect, defaultTier: tier, defaultTimeoutSeconds: timeout,175                                maxOutputBytes: maxOutputBytes, concurrency: concurrency, supportsDryRun: supportsDryRun,176                                idempotent: idempotent || sideEffect == .read, redactedInputKeys: redactedKeys, outputIsUntrusted: untrusted)177        self.tier = tier178        self.body = body179    }180181    func requirements(for input: ToolInput, context: ToolExecutionContext) throws -> ToolRequirements {182        ToolRequirements(capabilities: try capabilities(input, context), tier: tier ?? manifest.defaultTier, summary: "Probe \(manifest.identifier)")183    }184    func execute(input: ToolInput, context: ToolExecutionContext) async throws -> ToolResult { try await body(input, context) }185}186187/// Executor + sinks over a temp directory with every built-in tool registered.188struct Harness {189    let tmp: TempDirectory190    let registry: ToolRegistry191    let executor: ToolExecutor192    let events = CollectingEventSink()193    let audit = CollectingAuditSink()194    let redactor = Redactor()195196    init(policy: (any PolicyDeciding)? = nil, approvals: any ApprovalBroker = ScriptedApprovalBroker(), extra: [any ORIXTool] = [],197         secrets: (any SecretResolving)? = nil, builtIns: Bool = true, pythonEnvRoot: URL? = nil, directory: TempDirectory? = nil) async throws {198        tmp = try directory ?? TempDirectory(prefix: "orix-tools")199        registry = ToolRegistry()200        if builtIns { try await registry.registerAll(ORIXToolRegistryTools.all(pythonEnvRoot: pythonEnvRoot ?? tmp.url.appendingPathComponent("py-env-unused"))) }201        try await registry.registerAll(extra)202        // By default: everything allowed, subprocesses confined to this temp directory, no network.203        let effective = policy ?? ConfinedAllowPolicy(confinement: Confinement(readRoots: [tmp.url.path], writeRoots: [tmp.url.path]))204        executor = ToolExecutor(registry: registry, policy: effective, approvals: approvals, audit: audit, events: events, secrets: secrets, redactor: redactor)205    }206207    @discardableResult208    func call(_ toolID: String, _ arguments: JSONValue = [:], dryRun: Bool = false, key: String? = nil, runID: String = "run-test",209              mode: ExecutionMode = .guided, maxToolSeconds: Double = 600, id: String = UUID().uuidString) async -> ToolResult {210        var context = tmp.invocation(runID: runID, mode: mode)211        context.maxToolSeconds = maxToolSeconds212        return await executor.invoke(ToolCall(id: id, toolID: toolID, arguments: arguments, dryRun: dryRun, idempotencyKey: key), context: context)213    }214215    func path(_ name: String) -> String { tmp.url.appendingPathComponent(name).path }216    func exists(_ name: String) -> Bool { FileManager.default.fileExists(atPath: path(name)) }217    func read(_ name: String) -> String { (try? String(contentsOfFile: path(name), encoding: .utf8)) ?? "" }218}219220enum TestProcess {221    /// Runs a helper process directly (test setup only, unconfined).222    @discardableResult223    static func run(_ executable: String, _ args: [String], cwd: String, env: [String: String] = [:]) async throws -> ProcessOutput {224        try await ProcessRunner().run(ProcessSpec(executable: executable, arguments: args, environment: env, workingDirectory: cwd, timeoutSeconds: 60))225    }226227    static func isAlive(_ pid: Int32) -> Bool {228        if kill(pid, 0) == 0 { return true }229        return errno == EPERM230    }231232    /// Waits until `condition` holds or the timeout expires.233    static func eventually(timeout: Double = 5, _ condition: () -> Bool) async -> Bool {234        let deadline = Date().addingTimeInterval(timeout)235        while Date() < deadline {236            if condition() { return true }237            try? await Task.sleep(for: .milliseconds(25))238        }239        return condition()240    }241242    static func pid(inFile path: String) -> Int32? {243        (try? String(contentsOfFile: path, encoding: .utf8)).flatMap { Int32($0.trimmingCharacters(in: .whitespacesAndNewlines)) }244    }245246    /// Creates a throwaway git repository with one commit.247    static func makeRepository(at dir: String) async throws {248        try FileManager.default.createDirectory(atPath: dir, withIntermediateDirectories: true)249        let env = ["GIT_CONFIG_GLOBAL": "/dev/null", "GIT_CONFIG_NOSYSTEM": "1"]250        try await run("/usr/bin/git", ["init", "-q", "-b", "main", dir], cwd: dir, env: env).requireSuccess(timeoutSeconds: 60)251        for (k, v) in [("user.name", "ORIX Test"), ("user.email", "test@example.invalid"), ("commit.gpgsign", "false")] {252            try await run("/usr/bin/git", ["-C", dir, "config", k, v], cwd: dir).requireSuccess(timeoutSeconds: 60)253        }254        try "hello\n".write(toFile: dir + "/README.md", atomically: true, encoding: .utf8)255        try await run("/usr/bin/git", ["-C", dir, "add", "README.md"], cwd: dir).requireSuccess(timeoutSeconds: 60)256        try await run("/usr/bin/git", ["-C", dir, "commit", "-q", "-m", "initial"], cwd: dir).requireSuccess(timeoutSeconds: 60)257    }258}259
Files main
Menu

Theme