SPB Git

Repository Public License

PythonScriptSystemTests.swift 20.5 KB 330 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX OS Automation9// File             : Packages/ORIXToolRegistry/Tests/ORIXToolRegistryTests/PythonScriptSystemTests.swift10// Description      : Integration tests of python.* (a managed environment in a11//                    temp root, no network installs), script.swift,12//                    script.javascript and system.*, plus unit tests of13//                    requirement pinning.14// Module           : ORIXToolRegistryTests15// Repository       : https://git.spboucher.ai/orix-automation16// Created          : 2026-11-2117// Last Modified    : 2026-11-2118//19// =============================================================================20// AUTHOR & PROFESSIONAL AFFILIATION21// =============================================================================22//23// Author           : Simon-Pierre Boucher24// Position         : Professor of Finance25// Institution      : Université du Québec en Outaouais (UQO)26// Department       : Département des sciences administratives27// Organization     : SP Boucher28//29// Website          : https://spboucher.ai30// Git              : https://git.spboucher.ai31//32// =============================================================================33// OFFICIAL CONTACT DIRECTORY34// =============================================================================35//36// General          : info@spboucher.ai37// Professional     : contact@spboucher.ai38// Direct           : simon@spboucher.ai39// Research         : research@spboucher.ai40// Development      : dev@spboucher.ai41// Support          : support@spboucher.ai42// Security/Admin   : admin@spboucher.ai43// Notifications    : notifications@spboucher.ai44//45// Preferred Contact: support@spboucher.ai46//47// =============================================================================48// COPYRIGHT & INTELLECTUAL PROPERTY49// =============================================================================50//51// Copyright (c) 202652// Simon-Pierre Boucher53//54// All rights reserved where applicable.55//56// This file is governed by the applicable project license,57// copyright notices, institutional agreements and relevant law.58//59// Unless expressly authorized by the applicable license or60// rights holder, copying, redistribution, sublicensing,61// publication and commercial exploitation are not permitted.62//63// Existing third-party rights and notices remain unaffected.64//65// =============================================================================66// CONFIDENTIALITY & SECURITY67// =============================================================================68//69// Classification   : Public70// Distribution     : Public repository; use is governed by the project license71// Security Contact : admin@spboucher.ai72//73// Access to this source does not independently grant permission74// to redistribute or disclose confidential materials.75//76// Do not embed credentials, private keys, access tokens,77// certificates or confidential personal information.78//79// =============================================================================80// LICENSE & MAINTENANCE81// =============================================================================82//83// License          : Proprietary, all rights reserved (see license file)84// License File     : LICENSE85// Maintainer       : Simon-Pierre Boucher86// Technical Contact: dev@spboucher.ai87//88// =============================================================================89// END OF SP BOUCHER SOURCE HEADER90// =============================================================================9192import Foundation93import Testing94import ORIXCore95import ORIXTestingSupport96@testable import ORIXToolRegistry9798@Suite("python.* tools (integration, real venv in a temp root)")99struct PythonToolsTests {100    @Test func managedEnvironmentLifecycleRunAndConfinement() async throws {101        guard PythonEnvironment.basePython() != nil else {102            print("SKIPPED(no python3 on the safe PATH: python.run cannot create its environment on this Mac)")103            return104        }105        let envHome = try TempDirectory(prefix: "orix-pyenv"); defer { envHome.remove() }106        let envRoot = envHome.url.appendingPathComponent("python-env")107        let h = try await Harness(pythonEnvRoot: envRoot); defer { h.tmp.remove() }108        let outside = try TempDirectory(prefix: "orix-outside"); defer { outside.remove() }109110        let before = await h.call("python.envInfo")111        #expect(before.status == .succeeded && before.output["exists"] == false && before.output["basePython"]?.stringValue != nil)112        #expect(before.output["baseVersion"]?.stringValue?.first?.isNumber == true && !FileManager.default.fileExists(atPath: envRoot.path), "envInfo never creates the environment")113114        let code = """115        import sys, os, json116        open('result.json', 'w').write(json.dumps({'sum': sum(range(10))}))117        open(os.path.join(os.environ['ORIX_ARTIFACTS_DIR'], 'chart.txt'), 'w').write('chart')118        print('venv' if sys.prefix != sys.base_prefix else 'system')119        print(sys.executable)120        print(sys.argv[1:])121        """122        let run = await h.call("python.run", ["code": .string(code), "args": ["--n", "3"]])123        #expect(run.status == .succeeded && run.output["exitCode"] == 0, "\(run.failure?.message ?? "") \(run.output["stderr"]?.stringValue ?? "")")124        let lines = (run.output["stdout"]?.stringValue ?? "").split(separator: "\n").map(String.init)125        #expect(lines.count == 3 && lines[0] == "venv" && lines[1].hasPrefix(envRoot.path) && lines[2] == "['--n', '3']")126        let files = Set(run.artifacts.map { ($0.path as NSString).lastPathComponent })127        #expect(files == ["result.json", "chart.txt"], "new files are returned as artifacts, the script itself is not: \(files)")128        #expect(run.artifacts.allSatisfy { $0.path.hasPrefix(h.path("artifacts")) && $0.sha256 != nil } && run.untrusted)129        #expect(FileManager.default.isExecutableFile(atPath: envRoot.path + "/bin/python"))130131        let after = await h.call("python.envInfo")132        #expect(after.output["exists"] == true && after.output["version"]?.stringValue?.first?.isNumber == true)133        #expect(after.output["packages"]?.arrayValue?.contains { $0["name"] == "pip" } == true, "\(after.output.serialized())")134135        // Script file + cwd + stdin.136        try h.tmp.write("work/tool.py", "import sys\ndata = sys.stdin.read()\nopen('out.txt', 'w').write(data.upper())\nprint(len(data))\n")137        let script = await h.call("python.run", ["path": "work/tool.py", "cwd": "work", "stdin": "abc"])138        #expect(script.output["stdout"] == "3\n" && h.read("work/out.txt") == "ABC" && script.artifacts.map(\.path) == [h.path("work/out.txt")])139140        // Confinement: no write outside the granted folder, no network, no credential stores.141        let probe = """142        import socket, os143        for label, action in [('write', lambda: open(r'\(outside.url.path)/leak.txt', 'w').write('x')),144                              ('net', lambda: socket.socket().bind(('127.0.0.1', 0))),145                              ('ssh', lambda: os.listdir(os.path.expanduser('~/.ssh')))]:146            try:147                action(); print(label, 'ALLOWED')148            except Exception as e:149                print(label, 'DENIED', type(e).__name__)150        """151        let confined = await h.call("python.run", ["code": .string(probe)])152        let report = confined.output["stdout"]?.stringValue ?? ""153        #expect(report.contains("write DENIED") && report.contains("net DENIED") && !report.contains("ALLOWED"), "\(report)")154        if FileManager.default.fileExists(atPath: NSHomeDirectory() + "/.ssh") { #expect(report.contains("ssh DENIED PermissionError"), "\(report)") }155        #expect(!FileManager.default.fileExists(atPath: outside.file("leak.txt").path))156157        let failing = await h.call("python.run", ["code": "raise SystemExit('boom')"])158        #expect(failing.status == .succeeded && failing.output["exitCode"] == 1 && failing.output["stderr"]?.stringValue?.contains("boom") == true)159        let slow = await h.call("python.run", ["code": "import time\ntime.sleep(60)", "timeoutSeconds": 1])160        #expect(slow.status == .timedOut)161        // The system interpreter's site-packages were never touched: the environment is self-contained under its root.162        #expect(FileManager.default.fileExists(atPath: envRoot.path + "/pyvenv.cfg"))163    }164165    @Test func argumentsAndRequirementsAreValidatedBeforeAnythingRuns() async throws {166        let policy = ApprovalPolicy(denyTools: ["python.run"])167        let h = try await Harness(policy: policy); defer { h.tmp.remove() }168        #expect(await h.call("python.run", [:]).status == .invalidInput)169        #expect(await h.call("python.run", ["code": "print(1)", "path": "x.py"]).status == .invalidInput)170        #expect(await h.call("python.run", ["code": "  "]).status == .invalidInput)171        for bad in ["requests", "requests>=2.0", "git+https://github.com/psf/requests", "https://example.com/pkg.whl", "./local-pkg", "-r requirements.txt",172                    "--index-url=https://evil.example/simple", "pkg @ file:///tmp/pkg", "pkg[extra]==1.0", "pkg==1.0; python_version<'3'", "pkg==1.*", "../pkg==1.0", "pkg==1.0 --pre"] {173            #expect(await h.call("python.run", ["code": "print(1)", "requirements": [.string(bad)]]).status == .invalidInput, "\(bad)")174        }175        #expect(await policy.requests.isEmpty, "invalid requirements never reach the policy")176177        // A valid pinned install is a tier-2 call that needs both PyPI hosts; here the policy refuses it, so nothing is installed.178        let denied = await h.call("python.run", ["code": "print(1)", "requirements": ["requests==2.32.3", "numpy==2.1.0"]], id: "call-py")179        #expect(denied.status == .denied)180        let request = try #require(await policy.requests.last)181        #expect(request.tier == .consequential)182        // The default working folder lives in the run's artifacts, which the executor grants implicitly.183        #expect(request.capabilities == [.scriptRuntime(name: "python"), .network(host: "pypi.org"), .network(host: "files.pythonhosted.org")])184        #expect(request.summary.contains("requests==2.32.3"))185        _ = await h.call("python.run", ["path": "scripts/a.py", "cwd": "scripts"])186        #expect(await policy.requests.last?.capabilities == [.scriptRuntime(name: "python"), .fileRead(path: h.path("scripts")), .fileRead(path: h.path("scripts/a.py"))])187        #expect(!h.exists("py-env-unused"), "a refused call does not create the environment")188    }189190    @Test func requirementPinsOnly() throws {191        for ok in ["requests==2.32.3", "numpy==1.26.4", "typing_extensions==4.12.2", "ruamel.yaml==0.18.6", "torch==2.1.0rc1", "pkg==1.0.post1", "A==1"] {192            #expect(try PythonEnvironment.validate(requirement: ok) == ok, "\(ok)")193        }194        #expect(try PythonEnvironment.validate(requirement: "  pandas==2.2.2 ") == "pandas==2.2.2")195        for bad in ["", "pandas", "pandas==", "==1.0", "pandas===1.0", "pandas~=2.0", "pandas==2.0,<3", "svn+ssh://host/repo", "pkg==1.0@evil", "C:\\pkg==1.0", "pkg.whl", "~/pkg==1"] {196            #expect(throws: ToolError.self, "\(bad)") { try PythonEnvironment.validate(requirement: bad) }197        }198        #expect(PythonEnvironment().root.path.hasSuffix("/ORIX Automation/python-env"))199        #expect(PythonEnvironment(root: URL(fileURLWithPath: "/tmp/x")).interpreter.hasSuffix("/x/bin/python"))200    }201}202203@Suite("script.* tools (integration, real interpreters)")204struct ScriptToolsTests {205    @Test func swiftScriptRunsConfined() async throws {206        guard FileManager.default.isExecutableFile(atPath: "/usr/bin/swift") else { print("SKIPPED(/usr/bin/swift is missing)"); return }207        let h = try await Harness(); defer { h.tmp.remove() }208        let outside = try TempDirectory(prefix: "orix-outside"); defer { outside.remove() }209        let source = """210        import Foundation211        let args = CommandLine.arguments.dropFirst()212        print("sum=\\(args.compactMap { Int($0) }.reduce(0, +))")213        let artifacts = ProcessInfo.processInfo.environment["ORIX_ARTIFACTS_DIR"]!214        try "made by swift".write(toFile: artifacts + "/swift-out.txt", atomically: false, encoding: .utf8)215        do { try "leak".write(toFile: "\(outside.url.path)/leak.txt", atomically: false, encoding: .utf8); print("outside WRITTEN") }216        catch { print("outside DENIED") }217        """218        let r = await h.call("script.swift", ["script": .string(source), "args": ["4", "5", "x"]])219        guard r.output["exitCode"] == 0 else {220            let stderr = r.output["stderr"]?.stringValue ?? r.failure?.message ?? ""221            if stderr.contains("xcrun") || stderr.contains("developer tools") || stderr.contains("xcode-select") {222                print("SKIPPED(the Swift toolchain is not usable here: \(stderr.prefix(200)))"); return223            }224            Issue.record("script.swift failed: \(r.status) \(stderr.prefix(2000))"); return225        }226        let stdout = r.output["stdout"]?.stringValue ?? ""227        #expect(r.status == .succeeded && stdout.contains("sum=9") && stdout.contains("outside DENIED") && !stdout.contains("WRITTEN"))228        #expect(h.read("artifacts/swift-out.txt") == "made by swift" && r.artifacts.map { ($0.path as NSString).lastPathComponent } == ["swift-out.txt"])229        #expect(!FileManager.default.fileExists(atPath: outside.file("leak.txt").path))230        let broken = await h.call("script.swift", ["script": "let x: Int = \"not an int\""])231        #expect(broken.status == .succeeded && broken.output["exitCode"] != 0 && broken.output["stderr"]?.stringValue?.contains("error") == true)232    }233234    @Test func javascriptRunsOutOfProcessInJSC() async throws {235        guard FileManager.default.isExecutableFile(atPath: ScriptJavaScriptTool.jscPath) else {236            let h = try await Harness(); defer { h.tmp.remove() }237            #expect(await h.call("script.javascript", ["script": "print(1)"]).status == .unsupported)238            print("SKIPPED(the jsc helper is not present on this Mac; the tool answers unsupported)")239            return240        }241        let policy = ApprovalPolicy()242        let h = try await Harness(policy: policy, approvals: ScriptedApprovalBroker(fallback: .approveOnce)); defer { h.tmp.remove() }243        let source = """244        const data = [3, 1, 2].sort().map(x => x * 2);245        print(JSON.stringify({ data, args: arguments, hasProcess: typeof process, hasFetch: typeof fetch }));246        """247        let r = await h.call("script.javascript", ["script": .string(source), "args": ["one", "two"]], id: "call-js")248        #expect(r.status == .succeeded && r.output["exitCode"] == 0, "\(r.output.serialized()) \(r.failure?.message ?? "")")249        let parsed = try JSONValue.parse((r.output["stdout"]?.stringValue ?? "").trimmingCharacters(in: .whitespacesAndNewlines))250        #expect(parsed["data"] == [2, 4, 6] && parsed["args"] == ["one", "two"] && parsed["hasProcess"] == "undefined" && parsed["hasFetch"] == "undefined")251        let request = await policy.requests.last252        #expect(request?.tier == .consequential && request?.capabilities == [.scriptRuntime(name: "javascript")])253        let thrown = await h.call("script.javascript", ["script": "throw new Error('kaboom')"])254        #expect(thrown.output["exitCode"] != 0 && (thrown.output["stderr"]?.stringValue ?? "").contains("kaboom") || (thrown.output["stdout"]?.stringValue ?? "").contains("kaboom"))255        let loop = await h.call("script.javascript", ["script": "while (true) {}", "timeoutSeconds": 1])256        #expect(loop.status == .timedOut)257        #expect(await h.call("script.javascript", ["script": ""]).status == .invalidInput)258    }259}260261@Suite("system.* tools (integration, this Mac)")262struct SystemToolsTests {263    @Test func info() async throws {264        let h = try await Harness(); defer { h.tmp.remove() }265        let r = await h.call("system.info")266        #expect(r.status == .succeeded && !r.untrusted)267        let v = ProcessInfo.processInfo.operatingSystemVersion268        #expect(r.output["osVersion"] == .string("\(v.majorVersion).\(v.minorVersion).\(v.patchVersion)"))269        #expect(r.output["model"]?.stringValue?.isEmpty == false && r.output["model"] != "unknown" && r.output["cpu"] != "unknown")270        #expect(r.output["cores"] == .int(ProcessInfo.processInfo.processorCount) && (r.output["memoryBytes"]?.intValue ?? 0) >= 4_000_000_000)271        #expect((r.output["uptimeSeconds"]?.intValue ?? 0) > 0 && ["arm64", "x86_64"].contains(r.output["architecture"]?.stringValue ?? ""))272    }273274    @Test func processes() async throws {275        let h = try await Harness(); defer { h.tmp.remove() }276        let cpu = await h.call("system.processes", ["limit": 5])277        let list = cpu.output["processes"]?.arrayValue ?? []278        #expect(cpu.status == .succeeded && list.count == 5 && (cpu.output["total"]?.intValue ?? 0) > 20, "\(cpu.failure?.message ?? "")")279        #expect(list.allSatisfy { ($0["pid"]?.intValue ?? 0) > 0 && $0["name"]?.stringValue?.isEmpty == false && $0["user"]?.stringValue?.isEmpty == false })280        let values = list.compactMap { $0["cpuPercent"]?.doubleValue }281        #expect(values == values.sorted(by: >))282        let memory = await h.call("system.processes", ["sortBy": "memory", "limit": 3])283        let rss = (memory.output["processes"]?.arrayValue ?? []).compactMap { $0["residentBytes"]?.intValue }284        #expect(rss.count == 3 && rss == rss.sorted(by: >) && rss[0] > 1_000_000)285        let parsed = SystemProcessesTool.parse("  PID  PPID  %CPU %MEM    RSS USER             COMM\n    1     0   0.3  0.1  19088 root             launchd\n  612     1  12.5  1.4 234112 simon            Google Chrome Helper\nbroken line\n")286        #expect(parsed.count == 2 && parsed[1]["name"] == "Google Chrome Helper" && parsed[1]["cpuPercent"] == 12.5 && parsed[1]["residentBytes"] == .int(234112 * 1024) && parsed[0]["ppid"] == 0)287    }288289    @Test func diskUsage() async throws {290        let h = try await Harness(); defer { h.tmp.remove() }291        let r = await h.call("system.diskUsage")292        let volumes = r.output["volumes"]?.arrayValue ?? []293        #expect(r.status == .succeeded && !volumes.isEmpty)294        let root = try #require(volumes.first { $0["path"] == "/" })295        let total = root["totalBytes"]?.intValue ?? 0, available = root["availableBytes"]?.intValue ?? -1296        #expect(total > 10_000_000_000 && available >= 0 && available <= total && root["internal"] != nil)297        #expect((root["usedPercent"]?.doubleValue ?? -1) >= 0 && (root["usedPercent"]?.doubleValue ?? 101) <= 100)298    }299300    @Test func powerState() async throws {301        let h = try await Harness(); defer { h.tmp.remove() }302        let r = await h.call("system.powerState")303        #expect(r.status == .succeeded, "\(r.failure?.message ?? "")")304        let batteries = r.output["batteries"]?.arrayValue ?? []305        #expect(r.output["hasBattery"] == .bool(!batteries.isEmpty) && r.output["source"]?.stringValue?.isEmpty == false)306        if batteries.isEmpty { #expect(r.output["onACPower"] == true, "a Mac without a battery runs on AC power") }307        for b in batteries { #expect((0...100).contains(b["percent"]?.intValue ?? -1) && b["charging"]?.boolValue != nil) }308    }309310    @Test func networkState() async throws {311        let h = try await Harness(); defer { h.tmp.remove() }312        let r = await h.call("system.networkState")313        #expect(r.status == .succeeded, "\(r.failure?.message ?? "")")314        #expect(["satisfied", "unsatisfied", "requiresConnection"].contains(r.output["status"]?.stringValue ?? ""))315        #expect(r.output["online"] == .bool(r.output["status"] == "satisfied") && r.output["interfaces"]?.arrayValue != nil)316        if r.output["online"] == true { #expect(r.output["usedInterfaceTypes"]?.arrayValue?.isEmpty == false) }317    }318319    @Test func systemToolsNeedNoCapability() async throws {320        let policy = ApprovalPolicy()321        let h = try await Harness(policy: policy); defer { h.tmp.remove() }322        for id in ["system.info", "system.processes", "system.diskUsage", "system.powerState", "system.networkState", "python.envInfo"] {323            #expect(await h.call(id).status == .succeeded, "\(id)")324        }325        #expect(await policy.requests.count == 6)326        #expect(await policy.requests.allSatisfy { $0.capabilities.isEmpty && $0.tier == .readOnly })327        #expect(await h.call("system.info", ["extra": 1]).status == .invalidInput)328    }329}330
Files main
Menu

Theme