Repository Public License
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX OS Automation9// File : Packages/ORIXToolRegistry/Tests/ORIXToolRegistryTests/PythonScriptSystemTests.swift10// Description : Integration tests of python.* (a managed environment in a11// temp root, no network installs), script.swift,12// script.javascript and system.*, plus unit tests of13// requirement pinning.14// Module : ORIXToolRegistryTests15// Repository : https://git.spboucher.ai/orix-automation16// Created : 2026-11-2117// Last Modified : 2026-11-2118//19// =============================================================================20// AUTHOR & PROFESSIONAL AFFILIATION21// =============================================================================22//23// Author : Simon-Pierre Boucher24// Position : Professor of Finance25// Institution : Université du Québec en Outaouais (UQO)26// Department : Département des sciences administratives27// Organization : SP Boucher28//29// Website : https://spboucher.ai30// Git : https://git.spboucher.ai31//32// =============================================================================33// OFFICIAL CONTACT DIRECTORY34// =============================================================================35//36// General : info@spboucher.ai37// Professional : contact@spboucher.ai38// Direct : simon@spboucher.ai39// Research : research@spboucher.ai40// Development : dev@spboucher.ai41// Support : support@spboucher.ai42// Security/Admin : admin@spboucher.ai43// Notifications : notifications@spboucher.ai44//45// Preferred Contact: support@spboucher.ai46//47// =============================================================================48// COPYRIGHT & INTELLECTUAL PROPERTY49// =============================================================================50//51// Copyright (c) 202652// Simon-Pierre Boucher53//54// All rights reserved where applicable.55//56// This file is governed by the applicable project license,57// copyright notices, institutional agreements and relevant law.58//59// Unless expressly authorized by the applicable license or60// rights holder, copying, redistribution, sublicensing,61// publication and commercial exploitation are not permitted.62//63// Existing third-party rights and notices remain unaffected.64//65// =============================================================================66// CONFIDENTIALITY & SECURITY67// =============================================================================68//69// Classification : Public70// Distribution : Public repository; use is governed by the project license71// Security Contact : admin@spboucher.ai72//73// Access to this source does not independently grant permission74// to redistribute or disclose confidential materials.75//76// Do not embed credentials, private keys, access tokens,77// certificates or confidential personal information.78//79// =============================================================================80// LICENSE & MAINTENANCE81// =============================================================================82//83// License : Proprietary, all rights reserved (see license file)84// License File : LICENSE85// Maintainer : Simon-Pierre Boucher86// Technical Contact: dev@spboucher.ai87//88// =============================================================================89// END OF SP BOUCHER SOURCE HEADER90// =============================================================================9192import Foundation93import Testing94import ORIXCore95import ORIXTestingSupport96@testable import ORIXToolRegistry9798@Suite("python.* tools (integration, real venv in a temp root)")99struct PythonToolsTests {100 @Test func managedEnvironmentLifecycleRunAndConfinement() async throws {101 guard PythonEnvironment.basePython() != nil else {102 print("SKIPPED(no python3 on the safe PATH: python.run cannot create its environment on this Mac)")103 return104 }105 let envHome = try TempDirectory(prefix: "orix-pyenv"); defer { envHome.remove() }106 let envRoot = envHome.url.appendingPathComponent("python-env")107 let h = try await Harness(pythonEnvRoot: envRoot); defer { h.tmp.remove() }108 let outside = try TempDirectory(prefix: "orix-outside"); defer { outside.remove() }109110 let before = await h.call("python.envInfo")111 #expect(before.status == .succeeded && before.output["exists"] == false && before.output["basePython"]?.stringValue != nil)112 #expect(before.output["baseVersion"]?.stringValue?.first?.isNumber == true && !FileManager.default.fileExists(atPath: envRoot.path), "envInfo never creates the environment")113114 let code = """115 import sys, os, json116 open('result.json', 'w').write(json.dumps({'sum': sum(range(10))}))117 open(os.path.join(os.environ['ORIX_ARTIFACTS_DIR'], 'chart.txt'), 'w').write('chart')118 print('venv' if sys.prefix != sys.base_prefix else 'system')119 print(sys.executable)120 print(sys.argv[1:])121 """122 let run = await h.call("python.run", ["code": .string(code), "args": ["--n", "3"]])123 #expect(run.status == .succeeded && run.output["exitCode"] == 0, "\(run.failure?.message ?? "") \(run.output["stderr"]?.stringValue ?? "")")124 let lines = (run.output["stdout"]?.stringValue ?? "").split(separator: "\n").map(String.init)125 #expect(lines.count == 3 && lines[0] == "venv" && lines[1].hasPrefix(envRoot.path) && lines[2] == "['--n', '3']")126 let files = Set(run.artifacts.map { ($0.path as NSString).lastPathComponent })127 #expect(files == ["result.json", "chart.txt"], "new files are returned as artifacts, the script itself is not: \(files)")128 #expect(run.artifacts.allSatisfy { $0.path.hasPrefix(h.path("artifacts")) && $0.sha256 != nil } && run.untrusted)129 #expect(FileManager.default.isExecutableFile(atPath: envRoot.path + "/bin/python"))130131 let after = await h.call("python.envInfo")132 #expect(after.output["exists"] == true && after.output["version"]?.stringValue?.first?.isNumber == true)133 #expect(after.output["packages"]?.arrayValue?.contains { $0["name"] == "pip" } == true, "\(after.output.serialized())")134135 // Script file + cwd + stdin.136 try h.tmp.write("work/tool.py", "import sys\ndata = sys.stdin.read()\nopen('out.txt', 'w').write(data.upper())\nprint(len(data))\n")137 let script = await h.call("python.run", ["path": "work/tool.py", "cwd": "work", "stdin": "abc"])138 #expect(script.output["stdout"] == "3\n" && h.read("work/out.txt") == "ABC" && script.artifacts.map(\.path) == [h.path("work/out.txt")])139140 // Confinement: no write outside the granted folder, no network, no credential stores.141 let probe = """142 import socket, os143 for label, action in [('write', lambda: open(r'\(outside.url.path)/leak.txt', 'w').write('x')),144 ('net', lambda: socket.socket().bind(('127.0.0.1', 0))),145 ('ssh', lambda: os.listdir(os.path.expanduser('~/.ssh')))]:146 try:147 action(); print(label, 'ALLOWED')148 except Exception as e:149 print(label, 'DENIED', type(e).__name__)150 """151 let confined = await h.call("python.run", ["code": .string(probe)])152 let report = confined.output["stdout"]?.stringValue ?? ""153 #expect(report.contains("write DENIED") && report.contains("net DENIED") && !report.contains("ALLOWED"), "\(report)")154 if FileManager.default.fileExists(atPath: NSHomeDirectory() + "/.ssh") { #expect(report.contains("ssh DENIED PermissionError"), "\(report)") }155 #expect(!FileManager.default.fileExists(atPath: outside.file("leak.txt").path))156157 let failing = await h.call("python.run", ["code": "raise SystemExit('boom')"])158 #expect(failing.status == .succeeded && failing.output["exitCode"] == 1 && failing.output["stderr"]?.stringValue?.contains("boom") == true)159 let slow = await h.call("python.run", ["code": "import time\ntime.sleep(60)", "timeoutSeconds": 1])160 #expect(slow.status == .timedOut)161 // The system interpreter's site-packages were never touched: the environment is self-contained under its root.162 #expect(FileManager.default.fileExists(atPath: envRoot.path + "/pyvenv.cfg"))163 }164165 @Test func argumentsAndRequirementsAreValidatedBeforeAnythingRuns() async throws {166 let policy = ApprovalPolicy(denyTools: ["python.run"])167 let h = try await Harness(policy: policy); defer { h.tmp.remove() }168 #expect(await h.call("python.run", [:]).status == .invalidInput)169 #expect(await h.call("python.run", ["code": "print(1)", "path": "x.py"]).status == .invalidInput)170 #expect(await h.call("python.run", ["code": " "]).status == .invalidInput)171 for bad in ["requests", "requests>=2.0", "git+https://github.com/psf/requests", "https://example.com/pkg.whl", "./local-pkg", "-r requirements.txt",172 "--index-url=https://evil.example/simple", "pkg @ file:///tmp/pkg", "pkg[extra]==1.0", "pkg==1.0; python_version<'3'", "pkg==1.*", "../pkg==1.0", "pkg==1.0 --pre"] {173 #expect(await h.call("python.run", ["code": "print(1)", "requirements": [.string(bad)]]).status == .invalidInput, "\(bad)")174 }175 #expect(await policy.requests.isEmpty, "invalid requirements never reach the policy")176177 // A valid pinned install is a tier-2 call that needs both PyPI hosts; here the policy refuses it, so nothing is installed.178 let denied = await h.call("python.run", ["code": "print(1)", "requirements": ["requests==2.32.3", "numpy==2.1.0"]], id: "call-py")179 #expect(denied.status == .denied)180 let request = try #require(await policy.requests.last)181 #expect(request.tier == .consequential)182 // The default working folder lives in the run's artifacts, which the executor grants implicitly.183 #expect(request.capabilities == [.scriptRuntime(name: "python"), .network(host: "pypi.org"), .network(host: "files.pythonhosted.org")])184 #expect(request.summary.contains("requests==2.32.3"))185 _ = await h.call("python.run", ["path": "scripts/a.py", "cwd": "scripts"])186 #expect(await policy.requests.last?.capabilities == [.scriptRuntime(name: "python"), .fileRead(path: h.path("scripts")), .fileRead(path: h.path("scripts/a.py"))])187 #expect(!h.exists("py-env-unused"), "a refused call does not create the environment")188 }189190 @Test func requirementPinsOnly() throws {191 for ok in ["requests==2.32.3", "numpy==1.26.4", "typing_extensions==4.12.2", "ruamel.yaml==0.18.6", "torch==2.1.0rc1", "pkg==1.0.post1", "A==1"] {192 #expect(try PythonEnvironment.validate(requirement: ok) == ok, "\(ok)")193 }194 #expect(try PythonEnvironment.validate(requirement: " pandas==2.2.2 ") == "pandas==2.2.2")195 for bad in ["", "pandas", "pandas==", "==1.0", "pandas===1.0", "pandas~=2.0", "pandas==2.0,<3", "svn+ssh://host/repo", "pkg==1.0@evil", "C:\\pkg==1.0", "pkg.whl", "~/pkg==1"] {196 #expect(throws: ToolError.self, "\(bad)") { try PythonEnvironment.validate(requirement: bad) }197 }198 #expect(PythonEnvironment().root.path.hasSuffix("/ORIX Automation/python-env"))199 #expect(PythonEnvironment(root: URL(fileURLWithPath: "/tmp/x")).interpreter.hasSuffix("/x/bin/python"))200 }201}202203@Suite("script.* tools (integration, real interpreters)")204struct ScriptToolsTests {205 @Test func swiftScriptRunsConfined() async throws {206 guard FileManager.default.isExecutableFile(atPath: "/usr/bin/swift") else { print("SKIPPED(/usr/bin/swift is missing)"); return }207 let h = try await Harness(); defer { h.tmp.remove() }208 let outside = try TempDirectory(prefix: "orix-outside"); defer { outside.remove() }209 let source = """210 import Foundation211 let args = CommandLine.arguments.dropFirst()212 print("sum=\\(args.compactMap { Int($0) }.reduce(0, +))")213 let artifacts = ProcessInfo.processInfo.environment["ORIX_ARTIFACTS_DIR"]!214 try "made by swift".write(toFile: artifacts + "/swift-out.txt", atomically: false, encoding: .utf8)215 do { try "leak".write(toFile: "\(outside.url.path)/leak.txt", atomically: false, encoding: .utf8); print("outside WRITTEN") }216 catch { print("outside DENIED") }217 """218 let r = await h.call("script.swift", ["script": .string(source), "args": ["4", "5", "x"]])219 guard r.output["exitCode"] == 0 else {220 let stderr = r.output["stderr"]?.stringValue ?? r.failure?.message ?? ""221 if stderr.contains("xcrun") || stderr.contains("developer tools") || stderr.contains("xcode-select") {222 print("SKIPPED(the Swift toolchain is not usable here: \(stderr.prefix(200)))"); return223 }224 Issue.record("script.swift failed: \(r.status) \(stderr.prefix(2000))"); return225 }226 let stdout = r.output["stdout"]?.stringValue ?? ""227 #expect(r.status == .succeeded && stdout.contains("sum=9") && stdout.contains("outside DENIED") && !stdout.contains("WRITTEN"))228 #expect(h.read("artifacts/swift-out.txt") == "made by swift" && r.artifacts.map { ($0.path as NSString).lastPathComponent } == ["swift-out.txt"])229 #expect(!FileManager.default.fileExists(atPath: outside.file("leak.txt").path))230 let broken = await h.call("script.swift", ["script": "let x: Int = \"not an int\""])231 #expect(broken.status == .succeeded && broken.output["exitCode"] != 0 && broken.output["stderr"]?.stringValue?.contains("error") == true)232 }233234 @Test func javascriptRunsOutOfProcessInJSC() async throws {235 guard FileManager.default.isExecutableFile(atPath: ScriptJavaScriptTool.jscPath) else {236 let h = try await Harness(); defer { h.tmp.remove() }237 #expect(await h.call("script.javascript", ["script": "print(1)"]).status == .unsupported)238 print("SKIPPED(the jsc helper is not present on this Mac; the tool answers unsupported)")239 return240 }241 let policy = ApprovalPolicy()242 let h = try await Harness(policy: policy, approvals: ScriptedApprovalBroker(fallback: .approveOnce)); defer { h.tmp.remove() }243 let source = """244 const data = [3, 1, 2].sort().map(x => x * 2);245 print(JSON.stringify({ data, args: arguments, hasProcess: typeof process, hasFetch: typeof fetch }));246 """247 let r = await h.call("script.javascript", ["script": .string(source), "args": ["one", "two"]], id: "call-js")248 #expect(r.status == .succeeded && r.output["exitCode"] == 0, "\(r.output.serialized()) \(r.failure?.message ?? "")")249 let parsed = try JSONValue.parse((r.output["stdout"]?.stringValue ?? "").trimmingCharacters(in: .whitespacesAndNewlines))250 #expect(parsed["data"] == [2, 4, 6] && parsed["args"] == ["one", "two"] && parsed["hasProcess"] == "undefined" && parsed["hasFetch"] == "undefined")251 let request = await policy.requests.last252 #expect(request?.tier == .consequential && request?.capabilities == [.scriptRuntime(name: "javascript")])253 let thrown = await h.call("script.javascript", ["script": "throw new Error('kaboom')"])254 #expect(thrown.output["exitCode"] != 0 && (thrown.output["stderr"]?.stringValue ?? "").contains("kaboom") || (thrown.output["stdout"]?.stringValue ?? "").contains("kaboom"))255 let loop = await h.call("script.javascript", ["script": "while (true) {}", "timeoutSeconds": 1])256 #expect(loop.status == .timedOut)257 #expect(await h.call("script.javascript", ["script": ""]).status == .invalidInput)258 }259}260261@Suite("system.* tools (integration, this Mac)")262struct SystemToolsTests {263 @Test func info() async throws {264 let h = try await Harness(); defer { h.tmp.remove() }265 let r = await h.call("system.info")266 #expect(r.status == .succeeded && !r.untrusted)267 let v = ProcessInfo.processInfo.operatingSystemVersion268 #expect(r.output["osVersion"] == .string("\(v.majorVersion).\(v.minorVersion).\(v.patchVersion)"))269 #expect(r.output["model"]?.stringValue?.isEmpty == false && r.output["model"] != "unknown" && r.output["cpu"] != "unknown")270 #expect(r.output["cores"] == .int(ProcessInfo.processInfo.processorCount) && (r.output["memoryBytes"]?.intValue ?? 0) >= 4_000_000_000)271 #expect((r.output["uptimeSeconds"]?.intValue ?? 0) > 0 && ["arm64", "x86_64"].contains(r.output["architecture"]?.stringValue ?? ""))272 }273274 @Test func processes() async throws {275 let h = try await Harness(); defer { h.tmp.remove() }276 let cpu = await h.call("system.processes", ["limit": 5])277 let list = cpu.output["processes"]?.arrayValue ?? []278 #expect(cpu.status == .succeeded && list.count == 5 && (cpu.output["total"]?.intValue ?? 0) > 20, "\(cpu.failure?.message ?? "")")279 #expect(list.allSatisfy { ($0["pid"]?.intValue ?? 0) > 0 && $0["name"]?.stringValue?.isEmpty == false && $0["user"]?.stringValue?.isEmpty == false })280 let values = list.compactMap { $0["cpuPercent"]?.doubleValue }281 #expect(values == values.sorted(by: >))282 let memory = await h.call("system.processes", ["sortBy": "memory", "limit": 3])283 let rss = (memory.output["processes"]?.arrayValue ?? []).compactMap { $0["residentBytes"]?.intValue }284 #expect(rss.count == 3 && rss == rss.sorted(by: >) && rss[0] > 1_000_000)285 let parsed = SystemProcessesTool.parse(" PID PPID %CPU %MEM RSS USER COMM\n 1 0 0.3 0.1 19088 root launchd\n 612 1 12.5 1.4 234112 simon Google Chrome Helper\nbroken line\n")286 #expect(parsed.count == 2 && parsed[1]["name"] == "Google Chrome Helper" && parsed[1]["cpuPercent"] == 12.5 && parsed[1]["residentBytes"] == .int(234112 * 1024) && parsed[0]["ppid"] == 0)287 }288289 @Test func diskUsage() async throws {290 let h = try await Harness(); defer { h.tmp.remove() }291 let r = await h.call("system.diskUsage")292 let volumes = r.output["volumes"]?.arrayValue ?? []293 #expect(r.status == .succeeded && !volumes.isEmpty)294 let root = try #require(volumes.first { $0["path"] == "/" })295 let total = root["totalBytes"]?.intValue ?? 0, available = root["availableBytes"]?.intValue ?? -1296 #expect(total > 10_000_000_000 && available >= 0 && available <= total && root["internal"] != nil)297 #expect((root["usedPercent"]?.doubleValue ?? -1) >= 0 && (root["usedPercent"]?.doubleValue ?? 101) <= 100)298 }299300 @Test func powerState() async throws {301 let h = try await Harness(); defer { h.tmp.remove() }302 let r = await h.call("system.powerState")303 #expect(r.status == .succeeded, "\(r.failure?.message ?? "")")304 let batteries = r.output["batteries"]?.arrayValue ?? []305 #expect(r.output["hasBattery"] == .bool(!batteries.isEmpty) && r.output["source"]?.stringValue?.isEmpty == false)306 if batteries.isEmpty { #expect(r.output["onACPower"] == true, "a Mac without a battery runs on AC power") }307 for b in batteries { #expect((0...100).contains(b["percent"]?.intValue ?? -1) && b["charging"]?.boolValue != nil) }308 }309310 @Test func networkState() async throws {311 let h = try await Harness(); defer { h.tmp.remove() }312 let r = await h.call("system.networkState")313 #expect(r.status == .succeeded, "\(r.failure?.message ?? "")")314 #expect(["satisfied", "unsatisfied", "requiresConnection"].contains(r.output["status"]?.stringValue ?? ""))315 #expect(r.output["online"] == .bool(r.output["status"] == "satisfied") && r.output["interfaces"]?.arrayValue != nil)316 if r.output["online"] == true { #expect(r.output["usedInterfaceTypes"]?.arrayValue?.isEmpty == false) }317 }318319 @Test func systemToolsNeedNoCapability() async throws {320 let policy = ApprovalPolicy()321 let h = try await Harness(policy: policy); defer { h.tmp.remove() }322 for id in ["system.info", "system.processes", "system.diskUsage", "system.powerState", "system.networkState", "python.envInfo"] {323 #expect(await h.call(id).status == .succeeded, "\(id)")324 }325 #expect(await policy.requests.count == 6)326 #expect(await policy.requests.allSatisfy { $0.capabilities.isEmpty && $0.tier == .readOnly })327 #expect(await h.call("system.info", ["extra": 1]).status == .invalidInput)328 }329}330