SPB Git

Repository Public License

ProcessRunnerTests.swift 22.9 KB 396 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX OS Automation9// File             : Packages/ORIXToolRegistry/Tests/ORIXToolRegistryTests/ProcessRunnerTests.swift10// Description      : Integration tests of ProcessRunner with real subprocesses11//                    (argv, env, output bounds, timeout, cancellation,12//                    process-tree cleanup) and security tests of the Seatbelt13//                    profile.14// Module           : ORIXToolRegistryTests15// Repository       : https://git.spboucher.ai/orix-automation16// Created          : 2026-11-2117// Last Modified    : 2026-11-2118//19// =============================================================================20// AUTHOR & PROFESSIONAL AFFILIATION21// =============================================================================22//23// Author           : Simon-Pierre Boucher24// Position         : Professor of Finance25// Institution      : Université du Québec en Outaouais (UQO)26// Department       : Département des sciences administratives27// Organization     : SP Boucher28//29// Website          : https://spboucher.ai30// Git              : https://git.spboucher.ai31//32// =============================================================================33// OFFICIAL CONTACT DIRECTORY34// =============================================================================35//36// General          : info@spboucher.ai37// Professional     : contact@spboucher.ai38// Direct           : simon@spboucher.ai39// Research         : research@spboucher.ai40// Development      : dev@spboucher.ai41// Support          : support@spboucher.ai42// Security/Admin   : admin@spboucher.ai43// Notifications    : notifications@spboucher.ai44//45// Preferred Contact: support@spboucher.ai46//47// =============================================================================48// COPYRIGHT & INTELLECTUAL PROPERTY49// =============================================================================50//51// Copyright (c) 202652// Simon-Pierre Boucher53//54// All rights reserved where applicable.55//56// This file is governed by the applicable project license,57// copyright notices, institutional agreements and relevant law.58//59// Unless expressly authorized by the applicable license or60// rights holder, copying, redistribution, sublicensing,61// publication and commercial exploitation are not permitted.62//63// Existing third-party rights and notices remain unaffected.64//65// =============================================================================66// CONFIDENTIALITY & SECURITY67// =============================================================================68//69// Classification   : Public70// Distribution     : Public repository; use is governed by the project license71// Security Contact : admin@spboucher.ai72//73// Access to this source does not independently grant permission74// to redistribute or disclose confidential materials.75//76// Do not embed credentials, private keys, access tokens,77// certificates or confidential personal information.78//79// =============================================================================80// LICENSE & MAINTENANCE81// =============================================================================82//83// License          : Proprietary, all rights reserved (see license file)84// License File     : LICENSE85// Maintainer       : Simon-Pierre Boucher86// Technical Contact: dev@spboucher.ai87//88// =============================================================================89// END OF SP BOUCHER SOURCE HEADER90// =============================================================================9192import Foundation93import Testing94import ORIXCore95import ORIXTestingSupport96@testable import ORIXToolRegistry9798@Suite("ProcessRunner (integration, real subprocesses)")99struct ProcessRunnerTests {100    let runner = ProcessRunner()101102    func sh(_ script: String, cwd: String, timeout: Double = 20, maxBytes: Int = 1_048_576, grace: Double = 2,103            profile: SeatbeltProfile? = nil, env: [String: String] = [:]) async throws -> ProcessOutput {104        try await runner.run(ProcessSpec(executable: "/bin/sh", arguments: ["-c", script], environment: env, workingDirectory: cwd,105                                         timeoutSeconds: timeout, maxOutputBytes: maxBytes, confinement: profile, killGraceSeconds: grace))106    }107108    @Test func capturesOutputExitCodeAndWorkingDirectory() async throws {109        let tmp = try TempDirectory(); defer { tmp.remove() }110        let out = try await sh("pwd; echo problem >&2; exit 3", cwd: tmp.url.path)111        #expect(out.exitCode == 3 && out.termination == .exited && out.signal == nil)112        #expect(out.stdoutText.trimmingCharacters(in: .whitespacesAndNewlines) == tmp.url.path)113        #expect(out.stderrText == "problem\n")114        #expect(runner.liveProcessGroups.isEmpty)115    }116117    @Test func argumentsAreLiteralArgvElements() async throws {118        let tmp = try TempDirectory(); defer { tmp.remove() }119        let nasty = "a b; rm -rf ~ $(echo x) `id` 'q' \"d\" *"120        let out = try await runner.run(ProcessSpec(executable: "/usr/bin/printf", arguments: ["%s|", nasty, "second"], workingDirectory: tmp.url.path))121        #expect(out.stdoutText == nasty + "|second|")122    }123124    @Test func environmentIsMinimalAndExplicit() async throws {125        let tmp = try TempDirectory(); defer { tmp.remove() }126        setenv("ORIX_TEST_LEAK", "must-not-be-inherited", 1)127        let out = try await runner.run(ProcessSpec(executable: "/usr/bin/env", environment: ["EXTRA": "yes"], workingDirectory: tmp.url.path))128        let vars = Dictionary(uniqueKeysWithValues: out.stdoutText.split(separator: "\n").compactMap { line -> (String, String)? in129            guard let eq = line.firstIndex(of: "=") else { return nil }130            return (String(line[..<eq]), String(line[line.index(after: eq)...]))131        })132        #expect(vars["PATH"] == ProcessRunner.safePath)133        #expect(vars["HOME"] == NSHomeDirectory() && vars["EXTRA"] == "yes" && vars["LANG"] != nil)134        #expect(vars["TMPDIR"]?.contains("orix-proc-") == true)135        #expect(vars["ORIX_TEST_LEAK"] == nil && vars["USER"] == nil && vars["SSH_AUTH_SOCK"] == nil)136        #expect(Set(vars.keys).subtracting(["PATH", "HOME", "LANG", "TMPDIR", "EXTRA", "__CF_USER_TEXT_ENCODING"]).isEmpty)137        // The scratch directory is removed when the process ends.138        #expect(!FileManager.default.fileExists(atPath: vars["TMPDIR"] ?? "/"))139    }140141    @Test func executableResolutionUsesOnlyTheSafePath() async throws {142        let tmp = try TempDirectory(); defer { tmp.remove() }143        #expect(ProcessRunner.resolveExecutable("ls") == "/bin/ls")144        #expect(ProcessRunner.resolveExecutable("git") == "/usr/bin/git")145        #expect(ProcessRunner.resolveExecutable("definitely-not-a-program-xyz") == nil)146        #expect(ProcessRunner.resolveExecutable("./ls") == nil && ProcessRunner.resolveExecutable("bin/ls") == nil)147        #expect(ProcessRunner.resolveExecutable("/bin") == nil && ProcessRunner.resolveExecutable("") == nil)148        // A program reachable only through a caller-supplied PATH is not found.149        let fake = try tmp.write("orixfake", "#!/bin/sh\necho hijacked\n")150        try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: fake.path)151        await #expect(throws: ToolError.self) {152            _ = try await runner.run(ProcessSpec(executable: "orixfake", environment: ["PATH": tmp.url.path], workingDirectory: tmp.url.path))153        }154        await #expect(throws: ToolError.self) {155            _ = try await runner.run(ProcessSpec(executable: "/bin/ls", workingDirectory: tmp.url.path + "/missing"))156        }157    }158159    @Test func standardInputIsDelivered() async throws {160        let tmp = try TempDirectory(); defer { tmp.remove() }161        let out = try await runner.run(ProcessSpec(executable: "/bin/cat", workingDirectory: tmp.url.path, stdin: Data("from stdin".utf8)))162        #expect(out.stdoutText == "from stdin")163        // Without stdin the child reads /dev/null and ends at once.164        let none = try await runner.run(ProcessSpec(executable: "/bin/cat", workingDirectory: tmp.url.path, timeoutSeconds: 5))165        #expect(none.termination == .exited && none.stdout.isEmpty)166    }167168    @Test func outputIsBoundedAndStreamed() async throws {169        let tmp = try TempDirectory(); defer { tmp.remove() }170        let chunks = Counter()171        var spec = ProcessSpec(executable: "/bin/sh", arguments: ["-c", "head -c 300000 /dev/zero | tr '\\0' 'a'; echo err >&2"],172                               workingDirectory: tmp.url.path, maxOutputBytes: 1_000)173        spec.onOutput = { stream, text in if stream == .stdout, !text.isEmpty { chunks.increment() } }174        let out = try await runner.run(spec)175        #expect(out.stdout.count == 1_000 && out.stdoutTruncated && !out.stderrTruncated && out.truncated)176        #expect(out.exitCode == 0 && chunks.value >= 1)177    }178179    @Test func timeoutTerminatesAndReports() async throws {180        let tmp = try TempDirectory(); defer { tmp.remove() }181        let clock = ContinuousClock(); let start = clock.now182        let out = try await runner.run(ProcessSpec(executable: "/bin/sleep", arguments: ["30"], workingDirectory: tmp.url.path, timeoutSeconds: 0.4))183        #expect(out.termination == .timedOut && out.signal == SIGTERM && out.exitCode == 128 + SIGTERM)184        #expect(clock.now - start < .seconds(5))185        #expect(throws: ToolError.timeout(seconds: 0.4)) { try out.throwIfInterrupted(timeoutSeconds: 0.4) }186    }187188    @Test func sigtermResistantGroupIsKilled() async throws {189        let tmp = try TempDirectory(); defer { tmp.remove() }190        let runner = self.runner191        // The script ignores SIGTERM (and so do its children); it is stopped once it has announced itself.192        let task = Task { try await runner.run(ProcessSpec(executable: "/bin/sh", arguments: ["-c", "trap '' TERM; echo $$ > leader.pid; while true; do sleep 1; done"],193                                                           workingDirectory: tmp.url.path, timeoutSeconds: 120, killGraceSeconds: 0.4)) }194        #expect(await TestProcess.eventually(timeout: 15) { TestProcess.pid(inFile: tmp.file("leader.pid").path) != nil })195        let pid = try #require(TestProcess.pid(inFile: tmp.file("leader.pid").path))196        try await Task.sleep(for: .milliseconds(100))197        task.cancel()198        let out = try await task.value199        #expect(out.termination == .canceled && out.signal == SIGKILL, "SIGTERM was ignored, so the group had to be killed")200        #expect(!TestProcess.isAlive(pid))201    }202203    static let forkingScript = "(sleep 300 & echo $! > grandchild.pid; wait) & echo $! > child.pid; wait"204205    @Test func timeoutKillsChildrenAndGrandchildren() async throws {206        let tmp = try TempDirectory(); defer { tmp.remove() }207        let out = try await sh(Self.forkingScript, cwd: tmp.url.path, timeout: 3, grace: 0.5)208        #expect(out.termination == .timedOut)209        let child = try #require(TestProcess.pid(inFile: tmp.file("child.pid").path))210        let grandchild = try #require(TestProcess.pid(inFile: tmp.file("grandchild.pid").path))211        #expect(await TestProcess.eventually(timeout: 3) { !TestProcess.isAlive(child) && !TestProcess.isAlive(grandchild) && !TestProcess.isAlive(out.pid) })212    }213214    @Test func cancellationKillsTheWholeTree() async throws {215        let tmp = try TempDirectory(); defer { tmp.remove() }216        let runner = self.runner217        let task = Task { try await runner.run(ProcessSpec(executable: "/bin/sh", arguments: ["-c", Self.forkingScript], workingDirectory: tmp.url.path,218                                                           timeoutSeconds: 120, killGraceSeconds: 0.5)) }219        #expect(await TestProcess.eventually { FileManager.default.fileExists(atPath: tmp.file("grandchild.pid").path) && TestProcess.pid(inFile: tmp.file("grandchild.pid").path) != nil })220        let grandchild = try #require(TestProcess.pid(inFile: tmp.file("grandchild.pid").path))221        let child = try #require(TestProcess.pid(inFile: tmp.file("child.pid").path))222        #expect(TestProcess.isAlive(grandchild))223        #expect(runner.liveProcessGroups.count == 1)224        task.cancel()225        let out = try await task.value226        #expect(out.termination == .canceled)227        #expect(throws: ToolError.canceled) { try out.throwIfInterrupted(timeoutSeconds: 1) }228        #expect(await TestProcess.eventually(timeout: 3) { !TestProcess.isAlive(child) && !TestProcess.isAlive(grandchild) })229        #expect(runner.liveProcessGroups.isEmpty)230    }231232    @Test func backgroundChildDoesNotOutliveANormalExit() async throws {233        let tmp = try TempDirectory(); defer { tmp.remove() }234        let clock = ContinuousClock(); let start = clock.now235        let out = try await sh("sleep 300 & echo $! > bg.pid; echo done", cwd: tmp.url.path)236        #expect(out.exitCode == 0 && out.stdoutText == "done\n" && out.termination == .exited)237        #expect(clock.now - start < .seconds(5))238        let bg = try #require(TestProcess.pid(inFile: tmp.file("bg.pid").path))239        #expect(await TestProcess.eventually(timeout: 3) { !TestProcess.isAlive(bg) })240    }241242    @Test func descendantThatLeftTheGroupIsKilledOnTimeout() async throws {243        let tmp = try TempDirectory(); defer { tmp.remove() }244        let code = """245        import os, time246        pid = os.fork()247        if pid == 0:248            os.setsid()249            open('escaped.pid', 'w').write(str(os.getpid()))250            time.sleep(300)251        else:252            time.sleep(300)253        """254        let out = try await runner.run(ProcessSpec(executable: "/usr/bin/python3", arguments: ["-c", code], workingDirectory: tmp.url.path,255                                                   timeoutSeconds: 5, killGraceSeconds: 0.5))256        #expect(out.termination == .timedOut)257        guard let escaped = TestProcess.pid(inFile: tmp.file("escaped.pid").path) else {258            print("SKIPPED(python3 did not start within the timeout, the escape scenario was not exercised)")259            return260        }261        #expect(await TestProcess.eventually(timeout: 3) { !TestProcess.isAlive(escaped) })262    }263264    @Test func killAllStopsEveryGroupOfThisRunner() async throws {265        let tmp = try TempDirectory(); defer { tmp.remove() }266        let runner = ProcessRunner()267        async let a = runner.run(ProcessSpec(executable: "/bin/sleep", arguments: ["60"], workingDirectory: tmp.url.path, killGraceSeconds: 0.5))268        async let b = runner.run(ProcessSpec(executable: "/bin/sleep", arguments: ["61"], workingDirectory: tmp.url.path, killGraceSeconds: 0.5))269        #expect(await TestProcess.eventually { runner.liveProcessGroups.count == 2 })270        runner.killAll()271        let (ra, rb) = try await (a, b)272        #expect(ra.termination == .killed && rb.termination == .killed)273        #expect(runner.liveProcessGroups.isEmpty)274    }275276    @Test func preCancelledTaskStartsNothing() async throws {277        let tmp = try TempDirectory(); defer { tmp.remove() }278        let runner = self.runner279        let task = Task { () -> Bool in280            while !Task.isCancelled { await Task.yield() }281            do { _ = try await runner.run(ProcessSpec(executable: "/usr/bin/touch", arguments: ["ran.txt"], workingDirectory: tmp.url.path)); return false }282            catch { return (error as? ToolError) == .canceled }283        }284        task.cancel()285        #expect(await task.value)286        #expect(!FileManager.default.fileExists(atPath: tmp.file("ran.txt").path))287    }288}289290@Suite("Seatbelt confinement (security, real sandbox-exec)")291struct SeatbeltConfinementTests {292    let runner = ProcessRunner()293294    func run(_ executable: String, _ args: [String], cwd: String, profile: SeatbeltProfile?) async throws -> ProcessOutput {295        try await runner.run(ProcessSpec(executable: executable, arguments: args, workingDirectory: cwd, timeoutSeconds: 60, confinement: profile))296    }297298    @Test func sandboxExecIsPresent() {299        #expect(SeatbeltProfile.isAvailable)300    }301302    @Test func writesAreLimitedToWriteRootsAndTheScratchDirectory() async throws {303        let tmp = try TempDirectory(); defer { tmp.remove() }304        let allowed = tmp.url.appendingPathComponent("allowed").path305        try FileManager.default.createDirectory(atPath: allowed, withIntermediateDirectories: true)306        let profile = SeatbeltProfile(writeRoots: [allowed], allowsNetwork: false)307        let script = """308        echo in > "\(allowed)/inside.txt" && echo INSIDE_OK309        echo out > "\(tmp.url.path)/outside.txt" 2>/dev/null && echo OUTSIDE_WRITTEN310        mkdir "\(tmp.url.path)/newdir" 2>/dev/null && echo MKDIR_DONE311        echo home > "$HOME/orix-seatbelt-test-\(UUID().uuidString).txt" 2>/dev/null && echo HOME_WRITTEN312        echo t > "$TMPDIR/scratch.txt" && cat "$TMPDIR/scratch.txt"313        rm -f "\(tmp.url.path)/victim.txt" 2>/dev/null314        true315        """316        try tmp.write("victim.txt", "keep me")317        let out = try await run("/bin/sh", ["-c", script], cwd: tmp.url.path, profile: profile)318        #expect(out.stdoutText.contains("INSIDE_OK") && out.stdoutText.contains("t\n"))319        #expect(!out.stdoutText.contains("OUTSIDE_WRITTEN") && !out.stdoutText.contains("MKDIR_DONE") && !out.stdoutText.contains("HOME_WRITTEN"))320        #expect(FileManager.default.fileExists(atPath: allowed + "/inside.txt"))321        #expect(!FileManager.default.fileExists(atPath: tmp.file("outside.txt").path) && !FileManager.default.fileExists(atPath: tmp.file("newdir").path))322        #expect(FileManager.default.fileExists(atPath: tmp.file("victim.txt").path), "a confined process must not delete outside its roots")323        // Control: the same script unconfined does write outside, so the denial above is the sandbox's doing.324        let control = try await run("/bin/sh", ["-c", "echo out > \"\(tmp.url.path)/outside.txt\" && echo OUTSIDE_WRITTEN"], cwd: tmp.url.path, profile: nil)325        #expect(control.stdoutText.contains("OUTSIDE_WRITTEN"))326    }327328    static let socketProbe = """329    import socket, sys330    try:331        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM); s.bind(('127.0.0.1', 0)); s.listen(1)332        c = socket.create_connection(s.getsockname(), timeout=3); print('NET_OK')333    except Exception as e:334        print('NET_DENIED', type(e).__name__)335    """336337    @Test func networkIsDeniedUnlessAllowed() async throws {338        let tmp = try TempDirectory(); defer { tmp.remove() }339        let denied = try await run("/usr/bin/python3", ["-c", Self.socketProbe], cwd: tmp.url.path, profile: SeatbeltProfile(writeRoots: [], allowsNetwork: false))340        #expect(denied.stdoutText.contains("NET_DENIED") && !denied.stdoutText.contains("NET_OK"), "\(denied.stdoutText) \(denied.stderrText)")341        let allowed = try await run("/usr/bin/python3", ["-c", Self.socketProbe], cwd: tmp.url.path, profile: SeatbeltProfile(writeRoots: [], allowsNetwork: true))342        #expect(allowed.stdoutText.contains("NET_OK"), "\(allowed.stdoutText) \(allowed.stderrText)")343        // Name resolution is networking too: a confined process cannot resolve a host (no DNS exfiltration).344        let dns = try await run("/usr/bin/python3", ["-c", "import socket\ntry:\n    socket.getaddrinfo('example.com', 443); print('RESOLVED')\nexcept Exception as e:\n    print('DNS_DENIED')"],345                                cwd: tmp.url.path, profile: SeatbeltProfile(writeRoots: [], allowsNetwork: false))346        #expect(dns.stdoutText.contains("DNS_DENIED"))347    }348349    @Test func credentialStoresAreUnreadableEvenWithRootWriteAccess() async throws {350        let tmp = try TempDirectory(); defer { tmp.remove() }351        let ssh = NSHomeDirectory() + "/.ssh"352        guard FileManager.default.fileExists(atPath: ssh) else {353            print("SKIPPED(~/.ssh does not exist on this Mac, the read denial cannot be observed)")354            return355        }356        let control = try await run("/bin/ls", [ssh], cwd: tmp.url.path, profile: nil)357        try #require(control.exitCode == 0, "control: ~/.ssh must be listable without the sandbox")358        for profile in [SeatbeltProfile(writeRoots: [], allowsNetwork: false), SeatbeltProfile(writeRoots: ["/"], allowsNetwork: true)] {359            let listing = try await run("/bin/ls", [ssh], cwd: tmp.url.path, profile: profile)360            #expect(listing.exitCode != 0 && listing.stdout.isEmpty)361            let cat = try await run("/bin/sh", ["-c", "cat \"$HOME\"/.ssh/* 2>/dev/null | wc -c"], cwd: tmp.url.path, profile: profile)362            #expect(cat.stdoutText.trimmingCharacters(in: .whitespacesAndNewlines) == "0")363            let keychains = try await run("/bin/ls", [NSHomeDirectory() + "/Library/Keychains"], cwd: tmp.url.path, profile: profile)364            #expect(keychains.exitCode != 0)365        }366    }367368    @Test func ordinaryToolsStillRunConfined() async throws {369        let tmp = try TempDirectory(); defer { tmp.remove() }370        try tmp.write("a.txt", "A")371        let profile = SeatbeltProfile(writeRoots: [tmp.url.path], allowsNetwork: false)372        let ls = try await run("/bin/ls", [tmp.url.path], cwd: tmp.url.path, profile: profile)373        #expect(ls.exitCode == 0 && ls.stdoutText.contains("a.txt"))374        let python = try await run("/usr/bin/python3", ["-c", "import json, hashlib; print(json.dumps({'ok': hashlib.sha256(b'x').hexdigest()[:4]}))"], cwd: tmp.url.path, profile: profile)375        #expect(python.exitCode == 0 && python.stdoutText.contains("\"ok\""), "\(python.stderrText)")376        let git = try await run("git", ["--version"], cwd: tmp.url.path, profile: profile)377        #expect(git.exitCode == 0 && git.stdoutText.hasPrefix("git version"), "\(git.stderrText)")378        let initRepo = try await run("git", ["init", "-q", "repo"], cwd: tmp.url.path, profile: profile)379        #expect(initRepo.exitCode == 0 && FileManager.default.fileExists(atPath: tmp.file("repo/.git/HEAD").path), "\(initRepo.stderrText)")380    }381382    @Test func profileSourceQuotesPathsAndOrdersRules() {383        let profile = SeatbeltProfile(writeRoots: ["/tmp/has \"quote\"", "relative/ignored"], allowsNetwork: false)384        let source = profile.source(tempDirectory: "/private/tmp/scratch")385        #expect(source.contains(#"(subpath "/private/tmp/has \"quote\"")"#))386        #expect(!source.contains("relative/ignored"))387        #expect(source.contains("(deny network*)") && source.contains("/private/tmp/scratch"))388        let denyWrites = try! #require(source.range(of: "(deny file-write*)"))389        let allowWrites = try! #require(source.range(of: "(allow file-write* (subpath"))390        let denyCredentials = try! #require(source.range(of: "(deny file-read* file-write*"))391        #expect(denyWrites.lowerBound < allowWrites.lowerBound && allowWrites.lowerBound < denyCredentials.lowerBound)392        #expect(!SeatbeltProfile(writeRoots: [], allowsNetwork: true).source().contains("deny network"))393        for root in SeatbeltProfile.deniedRoots { #expect(PathResolver.isProtected(root)) }394    }395}396
Files main
Menu

Theme