Repository Public License
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX OS Automation9// File : Packages/ORIXToolRegistry/Tests/ORIXToolRegistryTests/ProcessRunnerTests.swift10// Description : Integration tests of ProcessRunner with real subprocesses11// (argv, env, output bounds, timeout, cancellation,12// process-tree cleanup) and security tests of the Seatbelt13// profile.14// Module : ORIXToolRegistryTests15// Repository : https://git.spboucher.ai/orix-automation16// Created : 2026-11-2117// Last Modified : 2026-11-2118//19// =============================================================================20// AUTHOR & PROFESSIONAL AFFILIATION21// =============================================================================22//23// Author : Simon-Pierre Boucher24// Position : Professor of Finance25// Institution : Université du Québec en Outaouais (UQO)26// Department : Département des sciences administratives27// Organization : SP Boucher28//29// Website : https://spboucher.ai30// Git : https://git.spboucher.ai31//32// =============================================================================33// OFFICIAL CONTACT DIRECTORY34// =============================================================================35//36// General : info@spboucher.ai37// Professional : contact@spboucher.ai38// Direct : simon@spboucher.ai39// Research : research@spboucher.ai40// Development : dev@spboucher.ai41// Support : support@spboucher.ai42// Security/Admin : admin@spboucher.ai43// Notifications : notifications@spboucher.ai44//45// Preferred Contact: support@spboucher.ai46//47// =============================================================================48// COPYRIGHT & INTELLECTUAL PROPERTY49// =============================================================================50//51// Copyright (c) 202652// Simon-Pierre Boucher53//54// All rights reserved where applicable.55//56// This file is governed by the applicable project license,57// copyright notices, institutional agreements and relevant law.58//59// Unless expressly authorized by the applicable license or60// rights holder, copying, redistribution, sublicensing,61// publication and commercial exploitation are not permitted.62//63// Existing third-party rights and notices remain unaffected.64//65// =============================================================================66// CONFIDENTIALITY & SECURITY67// =============================================================================68//69// Classification : Public70// Distribution : Public repository; use is governed by the project license71// Security Contact : admin@spboucher.ai72//73// Access to this source does not independently grant permission74// to redistribute or disclose confidential materials.75//76// Do not embed credentials, private keys, access tokens,77// certificates or confidential personal information.78//79// =============================================================================80// LICENSE & MAINTENANCE81// =============================================================================82//83// License : Proprietary, all rights reserved (see license file)84// License File : LICENSE85// Maintainer : Simon-Pierre Boucher86// Technical Contact: dev@spboucher.ai87//88// =============================================================================89// END OF SP BOUCHER SOURCE HEADER90// =============================================================================9192import Foundation93import Testing94import ORIXCore95import ORIXTestingSupport96@testable import ORIXToolRegistry9798@Suite("ProcessRunner (integration, real subprocesses)")99struct ProcessRunnerTests {100 let runner = ProcessRunner()101102 func sh(_ script: String, cwd: String, timeout: Double = 20, maxBytes: Int = 1_048_576, grace: Double = 2,103 profile: SeatbeltProfile? = nil, env: [String: String] = [:]) async throws -> ProcessOutput {104 try await runner.run(ProcessSpec(executable: "/bin/sh", arguments: ["-c", script], environment: env, workingDirectory: cwd,105 timeoutSeconds: timeout, maxOutputBytes: maxBytes, confinement: profile, killGraceSeconds: grace))106 }107108 @Test func capturesOutputExitCodeAndWorkingDirectory() async throws {109 let tmp = try TempDirectory(); defer { tmp.remove() }110 let out = try await sh("pwd; echo problem >&2; exit 3", cwd: tmp.url.path)111 #expect(out.exitCode == 3 && out.termination == .exited && out.signal == nil)112 #expect(out.stdoutText.trimmingCharacters(in: .whitespacesAndNewlines) == tmp.url.path)113 #expect(out.stderrText == "problem\n")114 #expect(runner.liveProcessGroups.isEmpty)115 }116117 @Test func argumentsAreLiteralArgvElements() async throws {118 let tmp = try TempDirectory(); defer { tmp.remove() }119 let nasty = "a b; rm -rf ~ $(echo x) `id` 'q' \"d\" *"120 let out = try await runner.run(ProcessSpec(executable: "/usr/bin/printf", arguments: ["%s|", nasty, "second"], workingDirectory: tmp.url.path))121 #expect(out.stdoutText == nasty + "|second|")122 }123124 @Test func environmentIsMinimalAndExplicit() async throws {125 let tmp = try TempDirectory(); defer { tmp.remove() }126 setenv("ORIX_TEST_LEAK", "must-not-be-inherited", 1)127 let out = try await runner.run(ProcessSpec(executable: "/usr/bin/env", environment: ["EXTRA": "yes"], workingDirectory: tmp.url.path))128 let vars = Dictionary(uniqueKeysWithValues: out.stdoutText.split(separator: "\n").compactMap { line -> (String, String)? in129 guard let eq = line.firstIndex(of: "=") else { return nil }130 return (String(line[..<eq]), String(line[line.index(after: eq)...]))131 })132 #expect(vars["PATH"] == ProcessRunner.safePath)133 #expect(vars["HOME"] == NSHomeDirectory() && vars["EXTRA"] == "yes" && vars["LANG"] != nil)134 #expect(vars["TMPDIR"]?.contains("orix-proc-") == true)135 #expect(vars["ORIX_TEST_LEAK"] == nil && vars["USER"] == nil && vars["SSH_AUTH_SOCK"] == nil)136 #expect(Set(vars.keys).subtracting(["PATH", "HOME", "LANG", "TMPDIR", "EXTRA", "__CF_USER_TEXT_ENCODING"]).isEmpty)137 // The scratch directory is removed when the process ends.138 #expect(!FileManager.default.fileExists(atPath: vars["TMPDIR"] ?? "/"))139 }140141 @Test func executableResolutionUsesOnlyTheSafePath() async throws {142 let tmp = try TempDirectory(); defer { tmp.remove() }143 #expect(ProcessRunner.resolveExecutable("ls") == "/bin/ls")144 #expect(ProcessRunner.resolveExecutable("git") == "/usr/bin/git")145 #expect(ProcessRunner.resolveExecutable("definitely-not-a-program-xyz") == nil)146 #expect(ProcessRunner.resolveExecutable("./ls") == nil && ProcessRunner.resolveExecutable("bin/ls") == nil)147 #expect(ProcessRunner.resolveExecutable("/bin") == nil && ProcessRunner.resolveExecutable("") == nil)148 // A program reachable only through a caller-supplied PATH is not found.149 let fake = try tmp.write("orixfake", "#!/bin/sh\necho hijacked\n")150 try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: fake.path)151 await #expect(throws: ToolError.self) {152 _ = try await runner.run(ProcessSpec(executable: "orixfake", environment: ["PATH": tmp.url.path], workingDirectory: tmp.url.path))153 }154 await #expect(throws: ToolError.self) {155 _ = try await runner.run(ProcessSpec(executable: "/bin/ls", workingDirectory: tmp.url.path + "/missing"))156 }157 }158159 @Test func standardInputIsDelivered() async throws {160 let tmp = try TempDirectory(); defer { tmp.remove() }161 let out = try await runner.run(ProcessSpec(executable: "/bin/cat", workingDirectory: tmp.url.path, stdin: Data("from stdin".utf8)))162 #expect(out.stdoutText == "from stdin")163 // Without stdin the child reads /dev/null and ends at once.164 let none = try await runner.run(ProcessSpec(executable: "/bin/cat", workingDirectory: tmp.url.path, timeoutSeconds: 5))165 #expect(none.termination == .exited && none.stdout.isEmpty)166 }167168 @Test func outputIsBoundedAndStreamed() async throws {169 let tmp = try TempDirectory(); defer { tmp.remove() }170 let chunks = Counter()171 var spec = ProcessSpec(executable: "/bin/sh", arguments: ["-c", "head -c 300000 /dev/zero | tr '\\0' 'a'; echo err >&2"],172 workingDirectory: tmp.url.path, maxOutputBytes: 1_000)173 spec.onOutput = { stream, text in if stream == .stdout, !text.isEmpty { chunks.increment() } }174 let out = try await runner.run(spec)175 #expect(out.stdout.count == 1_000 && out.stdoutTruncated && !out.stderrTruncated && out.truncated)176 #expect(out.exitCode == 0 && chunks.value >= 1)177 }178179 @Test func timeoutTerminatesAndReports() async throws {180 let tmp = try TempDirectory(); defer { tmp.remove() }181 let clock = ContinuousClock(); let start = clock.now182 let out = try await runner.run(ProcessSpec(executable: "/bin/sleep", arguments: ["30"], workingDirectory: tmp.url.path, timeoutSeconds: 0.4))183 #expect(out.termination == .timedOut && out.signal == SIGTERM && out.exitCode == 128 + SIGTERM)184 #expect(clock.now - start < .seconds(5))185 #expect(throws: ToolError.timeout(seconds: 0.4)) { try out.throwIfInterrupted(timeoutSeconds: 0.4) }186 }187188 @Test func sigtermResistantGroupIsKilled() async throws {189 let tmp = try TempDirectory(); defer { tmp.remove() }190 let runner = self.runner191 // The script ignores SIGTERM (and so do its children); it is stopped once it has announced itself.192 let task = Task { try await runner.run(ProcessSpec(executable: "/bin/sh", arguments: ["-c", "trap '' TERM; echo $$ > leader.pid; while true; do sleep 1; done"],193 workingDirectory: tmp.url.path, timeoutSeconds: 120, killGraceSeconds: 0.4)) }194 #expect(await TestProcess.eventually(timeout: 15) { TestProcess.pid(inFile: tmp.file("leader.pid").path) != nil })195 let pid = try #require(TestProcess.pid(inFile: tmp.file("leader.pid").path))196 try await Task.sleep(for: .milliseconds(100))197 task.cancel()198 let out = try await task.value199 #expect(out.termination == .canceled && out.signal == SIGKILL, "SIGTERM was ignored, so the group had to be killed")200 #expect(!TestProcess.isAlive(pid))201 }202203 static let forkingScript = "(sleep 300 & echo $! > grandchild.pid; wait) & echo $! > child.pid; wait"204205 @Test func timeoutKillsChildrenAndGrandchildren() async throws {206 let tmp = try TempDirectory(); defer { tmp.remove() }207 let out = try await sh(Self.forkingScript, cwd: tmp.url.path, timeout: 3, grace: 0.5)208 #expect(out.termination == .timedOut)209 let child = try #require(TestProcess.pid(inFile: tmp.file("child.pid").path))210 let grandchild = try #require(TestProcess.pid(inFile: tmp.file("grandchild.pid").path))211 #expect(await TestProcess.eventually(timeout: 3) { !TestProcess.isAlive(child) && !TestProcess.isAlive(grandchild) && !TestProcess.isAlive(out.pid) })212 }213214 @Test func cancellationKillsTheWholeTree() async throws {215 let tmp = try TempDirectory(); defer { tmp.remove() }216 let runner = self.runner217 let task = Task { try await runner.run(ProcessSpec(executable: "/bin/sh", arguments: ["-c", Self.forkingScript], workingDirectory: tmp.url.path,218 timeoutSeconds: 120, killGraceSeconds: 0.5)) }219 #expect(await TestProcess.eventually { FileManager.default.fileExists(atPath: tmp.file("grandchild.pid").path) && TestProcess.pid(inFile: tmp.file("grandchild.pid").path) != nil })220 let grandchild = try #require(TestProcess.pid(inFile: tmp.file("grandchild.pid").path))221 let child = try #require(TestProcess.pid(inFile: tmp.file("child.pid").path))222 #expect(TestProcess.isAlive(grandchild))223 #expect(runner.liveProcessGroups.count == 1)224 task.cancel()225 let out = try await task.value226 #expect(out.termination == .canceled)227 #expect(throws: ToolError.canceled) { try out.throwIfInterrupted(timeoutSeconds: 1) }228 #expect(await TestProcess.eventually(timeout: 3) { !TestProcess.isAlive(child) && !TestProcess.isAlive(grandchild) })229 #expect(runner.liveProcessGroups.isEmpty)230 }231232 @Test func backgroundChildDoesNotOutliveANormalExit() async throws {233 let tmp = try TempDirectory(); defer { tmp.remove() }234 let clock = ContinuousClock(); let start = clock.now235 let out = try await sh("sleep 300 & echo $! > bg.pid; echo done", cwd: tmp.url.path)236 #expect(out.exitCode == 0 && out.stdoutText == "done\n" && out.termination == .exited)237 #expect(clock.now - start < .seconds(5))238 let bg = try #require(TestProcess.pid(inFile: tmp.file("bg.pid").path))239 #expect(await TestProcess.eventually(timeout: 3) { !TestProcess.isAlive(bg) })240 }241242 @Test func descendantThatLeftTheGroupIsKilledOnTimeout() async throws {243 let tmp = try TempDirectory(); defer { tmp.remove() }244 let code = """245 import os, time246 pid = os.fork()247 if pid == 0:248 os.setsid()249 open('escaped.pid', 'w').write(str(os.getpid()))250 time.sleep(300)251 else:252 time.sleep(300)253 """254 let out = try await runner.run(ProcessSpec(executable: "/usr/bin/python3", arguments: ["-c", code], workingDirectory: tmp.url.path,255 timeoutSeconds: 5, killGraceSeconds: 0.5))256 #expect(out.termination == .timedOut)257 guard let escaped = TestProcess.pid(inFile: tmp.file("escaped.pid").path) else {258 print("SKIPPED(python3 did not start within the timeout, the escape scenario was not exercised)")259 return260 }261 #expect(await TestProcess.eventually(timeout: 3) { !TestProcess.isAlive(escaped) })262 }263264 @Test func killAllStopsEveryGroupOfThisRunner() async throws {265 let tmp = try TempDirectory(); defer { tmp.remove() }266 let runner = ProcessRunner()267 async let a = runner.run(ProcessSpec(executable: "/bin/sleep", arguments: ["60"], workingDirectory: tmp.url.path, killGraceSeconds: 0.5))268 async let b = runner.run(ProcessSpec(executable: "/bin/sleep", arguments: ["61"], workingDirectory: tmp.url.path, killGraceSeconds: 0.5))269 #expect(await TestProcess.eventually { runner.liveProcessGroups.count == 2 })270 runner.killAll()271 let (ra, rb) = try await (a, b)272 #expect(ra.termination == .killed && rb.termination == .killed)273 #expect(runner.liveProcessGroups.isEmpty)274 }275276 @Test func preCancelledTaskStartsNothing() async throws {277 let tmp = try TempDirectory(); defer { tmp.remove() }278 let runner = self.runner279 let task = Task { () -> Bool in280 while !Task.isCancelled { await Task.yield() }281 do { _ = try await runner.run(ProcessSpec(executable: "/usr/bin/touch", arguments: ["ran.txt"], workingDirectory: tmp.url.path)); return false }282 catch { return (error as? ToolError) == .canceled }283 }284 task.cancel()285 #expect(await task.value)286 #expect(!FileManager.default.fileExists(atPath: tmp.file("ran.txt").path))287 }288}289290@Suite("Seatbelt confinement (security, real sandbox-exec)")291struct SeatbeltConfinementTests {292 let runner = ProcessRunner()293294 func run(_ executable: String, _ args: [String], cwd: String, profile: SeatbeltProfile?) async throws -> ProcessOutput {295 try await runner.run(ProcessSpec(executable: executable, arguments: args, workingDirectory: cwd, timeoutSeconds: 60, confinement: profile))296 }297298 @Test func sandboxExecIsPresent() {299 #expect(SeatbeltProfile.isAvailable)300 }301302 @Test func writesAreLimitedToWriteRootsAndTheScratchDirectory() async throws {303 let tmp = try TempDirectory(); defer { tmp.remove() }304 let allowed = tmp.url.appendingPathComponent("allowed").path305 try FileManager.default.createDirectory(atPath: allowed, withIntermediateDirectories: true)306 let profile = SeatbeltProfile(writeRoots: [allowed], allowsNetwork: false)307 let script = """308 echo in > "\(allowed)/inside.txt" && echo INSIDE_OK309 echo out > "\(tmp.url.path)/outside.txt" 2>/dev/null && echo OUTSIDE_WRITTEN310 mkdir "\(tmp.url.path)/newdir" 2>/dev/null && echo MKDIR_DONE311 echo home > "$HOME/orix-seatbelt-test-\(UUID().uuidString).txt" 2>/dev/null && echo HOME_WRITTEN312 echo t > "$TMPDIR/scratch.txt" && cat "$TMPDIR/scratch.txt"313 rm -f "\(tmp.url.path)/victim.txt" 2>/dev/null314 true315 """316 try tmp.write("victim.txt", "keep me")317 let out = try await run("/bin/sh", ["-c", script], cwd: tmp.url.path, profile: profile)318 #expect(out.stdoutText.contains("INSIDE_OK") && out.stdoutText.contains("t\n"))319 #expect(!out.stdoutText.contains("OUTSIDE_WRITTEN") && !out.stdoutText.contains("MKDIR_DONE") && !out.stdoutText.contains("HOME_WRITTEN"))320 #expect(FileManager.default.fileExists(atPath: allowed + "/inside.txt"))321 #expect(!FileManager.default.fileExists(atPath: tmp.file("outside.txt").path) && !FileManager.default.fileExists(atPath: tmp.file("newdir").path))322 #expect(FileManager.default.fileExists(atPath: tmp.file("victim.txt").path), "a confined process must not delete outside its roots")323 // Control: the same script unconfined does write outside, so the denial above is the sandbox's doing.324 let control = try await run("/bin/sh", ["-c", "echo out > \"\(tmp.url.path)/outside.txt\" && echo OUTSIDE_WRITTEN"], cwd: tmp.url.path, profile: nil)325 #expect(control.stdoutText.contains("OUTSIDE_WRITTEN"))326 }327328 static let socketProbe = """329 import socket, sys330 try:331 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM); s.bind(('127.0.0.1', 0)); s.listen(1)332 c = socket.create_connection(s.getsockname(), timeout=3); print('NET_OK')333 except Exception as e:334 print('NET_DENIED', type(e).__name__)335 """336337 @Test func networkIsDeniedUnlessAllowed() async throws {338 let tmp = try TempDirectory(); defer { tmp.remove() }339 let denied = try await run("/usr/bin/python3", ["-c", Self.socketProbe], cwd: tmp.url.path, profile: SeatbeltProfile(writeRoots: [], allowsNetwork: false))340 #expect(denied.stdoutText.contains("NET_DENIED") && !denied.stdoutText.contains("NET_OK"), "\(denied.stdoutText) \(denied.stderrText)")341 let allowed = try await run("/usr/bin/python3", ["-c", Self.socketProbe], cwd: tmp.url.path, profile: SeatbeltProfile(writeRoots: [], allowsNetwork: true))342 #expect(allowed.stdoutText.contains("NET_OK"), "\(allowed.stdoutText) \(allowed.stderrText)")343 // Name resolution is networking too: a confined process cannot resolve a host (no DNS exfiltration).344 let dns = try await run("/usr/bin/python3", ["-c", "import socket\ntry:\n socket.getaddrinfo('example.com', 443); print('RESOLVED')\nexcept Exception as e:\n print('DNS_DENIED')"],345 cwd: tmp.url.path, profile: SeatbeltProfile(writeRoots: [], allowsNetwork: false))346 #expect(dns.stdoutText.contains("DNS_DENIED"))347 }348349 @Test func credentialStoresAreUnreadableEvenWithRootWriteAccess() async throws {350 let tmp = try TempDirectory(); defer { tmp.remove() }351 let ssh = NSHomeDirectory() + "/.ssh"352 guard FileManager.default.fileExists(atPath: ssh) else {353 print("SKIPPED(~/.ssh does not exist on this Mac, the read denial cannot be observed)")354 return355 }356 let control = try await run("/bin/ls", [ssh], cwd: tmp.url.path, profile: nil)357 try #require(control.exitCode == 0, "control: ~/.ssh must be listable without the sandbox")358 for profile in [SeatbeltProfile(writeRoots: [], allowsNetwork: false), SeatbeltProfile(writeRoots: ["/"], allowsNetwork: true)] {359 let listing = try await run("/bin/ls", [ssh], cwd: tmp.url.path, profile: profile)360 #expect(listing.exitCode != 0 && listing.stdout.isEmpty)361 let cat = try await run("/bin/sh", ["-c", "cat \"$HOME\"/.ssh/* 2>/dev/null | wc -c"], cwd: tmp.url.path, profile: profile)362 #expect(cat.stdoutText.trimmingCharacters(in: .whitespacesAndNewlines) == "0")363 let keychains = try await run("/bin/ls", [NSHomeDirectory() + "/Library/Keychains"], cwd: tmp.url.path, profile: profile)364 #expect(keychains.exitCode != 0)365 }366 }367368 @Test func ordinaryToolsStillRunConfined() async throws {369 let tmp = try TempDirectory(); defer { tmp.remove() }370 try tmp.write("a.txt", "A")371 let profile = SeatbeltProfile(writeRoots: [tmp.url.path], allowsNetwork: false)372 let ls = try await run("/bin/ls", [tmp.url.path], cwd: tmp.url.path, profile: profile)373 #expect(ls.exitCode == 0 && ls.stdoutText.contains("a.txt"))374 let python = try await run("/usr/bin/python3", ["-c", "import json, hashlib; print(json.dumps({'ok': hashlib.sha256(b'x').hexdigest()[:4]}))"], cwd: tmp.url.path, profile: profile)375 #expect(python.exitCode == 0 && python.stdoutText.contains("\"ok\""), "\(python.stderrText)")376 let git = try await run("git", ["--version"], cwd: tmp.url.path, profile: profile)377 #expect(git.exitCode == 0 && git.stdoutText.hasPrefix("git version"), "\(git.stderrText)")378 let initRepo = try await run("git", ["init", "-q", "repo"], cwd: tmp.url.path, profile: profile)379 #expect(initRepo.exitCode == 0 && FileManager.default.fileExists(atPath: tmp.file("repo/.git/HEAD").path), "\(initRepo.stderrText)")380 }381382 @Test func profileSourceQuotesPathsAndOrdersRules() {383 let profile = SeatbeltProfile(writeRoots: ["/tmp/has \"quote\"", "relative/ignored"], allowsNetwork: false)384 let source = profile.source(tempDirectory: "/private/tmp/scratch")385 #expect(source.contains(#"(subpath "/private/tmp/has \"quote\"")"#))386 #expect(!source.contains("relative/ignored"))387 #expect(source.contains("(deny network*)") && source.contains("/private/tmp/scratch"))388 let denyWrites = try! #require(source.range(of: "(deny file-write*)"))389 let allowWrites = try! #require(source.range(of: "(allow file-write* (subpath"))390 let denyCredentials = try! #require(source.range(of: "(deny file-read* file-write*"))391 #expect(denyWrites.lowerBound < allowWrites.lowerBound && allowWrites.lowerBound < denyCredentials.lowerBound)392 #expect(!SeatbeltProfile(writeRoots: [], allowsNetwork: true).source().contains("deny network"))393 for root in SeatbeltProfile.deniedRoots { #expect(PathResolver.isProtected(root)) }394 }395}396