SPB Git

Repository Public

SecurityTests.swift 19.0 KB 305 lines swift

1import Foundation2import Testing3@testable import SheetPython45/// A TCP listener on 127.0.0.1 that accepts and immediately closes connections.6final class LocalListener: @unchecked Sendable {7    let port: Int8    private let fd: Int329    final class Counter: @unchecked Sendable { let lock = NSLock(); var n = 0; func increment() { lock.withLock { n += 1 } } }10    private let counter: Counter11    var connections: Int { counter.lock.withLock { counter.n } }1213    init() throws {14        let sock = socket(AF_INET, SOCK_STREAM, 0)15        var addr = sockaddr_in(); addr.sin_family = sa_family_t(AF_INET); addr.sin_addr.s_addr = inet_addr("127.0.0.1"); addr.sin_port = 016        addr.sin_len = UInt8(MemoryLayout<sockaddr_in>.size)17        let rc = withUnsafePointer(to: &addr) { $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { bind(sock, $0, socklen_t(MemoryLayout<sockaddr_in>.size)) } }18        guard rc == 0, listen(sock, 16) == 0 else { throw PythonHostError("cannot listen") }19        var bound = sockaddr_in(); var len = socklen_t(MemoryLayout<sockaddr_in>.size)20        _ = withUnsafeMutablePointer(to: &bound) { $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { getsockname(sock, $0, &len) } }21        port = Int(UInt16(bigEndian: bound.sin_port))22        fd = sock23        let counter = Counter()24        self.counter = counter25        Thread.detachNewThread {26            while true {27                let c = accept(sock, nil, nil)28                if c < 0 { break }29                counter.increment()30                var one: Int32 = 131                setsockopt(c, SOL_SOCKET, SO_NOSIGPIPE, &one, socklen_t(MemoryLayout<Int32>.size))32                _ = "HTTP/1.0 200 OK\r\nContent-Length: 2\r\n\r\nok".withCString { write(c, $0, strlen($0)) }33                close(c)34            }35        }36    }37    deinit { close(fd) }38}3940/// What a script observes when it attempts things restricted mode forbids. Each key maps to "ok" or the error.41let probeScript = """42import json, os, signal, socket, subprocess, sys, tempfile43PORT, OUTSIDE_TMP, DESKTOP, VICTIM, GRANT_R, GRANT_W = @PORT@, @OUTSIDE@, @DESKTOP@, @VICTIM@, @GR@, @GW@44out = {}45def attempt(name, f):46    try:47        f(); out[name] = "ok"48    except BaseException as e:49        out[name] = "%s: %s" % (type(e).__name__, e)50def connect():51    s = socket.create_connection(("127.0.0.1", PORT), timeout=3); s.close()52def udp():53    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); s.sendto(b"x", ("127.0.0.1", PORT)); s.close()54def listen():55    s = socket.socket(); s.bind(("127.0.0.1", 0)); s.listen(1); s.close()56def write(path):57    def f():58        with open(path, "w") as fh: fh.write("written by the script")59    return f60def curl():61    r = subprocess.run(["/usr/bin/curl", "-s", "-m", "3", "http://127.0.0.1:%d/" % PORT], capture_output=True)62    if r.returncode != 0: raise RuntimeError("curl exit %d" % r.returncode)63def spawn_self():64    r = subprocess.run([sys.executable, "-c", "print(6*7)"], capture_output=True, text=True)65    assert r.stdout.strip() == "42", r66def read_home():67    os.listdir(os.path.expanduser("~real"))68home = os.path.dirname(os.path.dirname(DESKTOP))69attempt("tcp_connect", connect)70attempt("udp_send", udp)71attempt("tcp_listen", listen)72attempt("write_desktop", write(DESKTOP))73attempt("write_outside_tmp", write(OUTSIDE_TMP))74attempt("write_run_dir", write(os.path.join(tempfile.gettempdir(), "inside.txt")))75attempt("write_cwd", write("cwd.txt"))76attempt("read_ssh", lambda: os.listdir(os.path.join(home, ".ssh")))77attempt("read_home_listing", lambda: os.listdir(home))78attempt("read_keychains", lambda: os.listdir(os.path.join(home, "Library", "Keychains")))79attempt("read_documents", lambda: os.listdir(os.path.join(home, "Documents")))80attempt("read_app_support", lambda: os.listdir(os.path.join(home, "Library", "Application Support")))81attempt("read_volumes", lambda: os.listdir("/Volumes"))82attempt("read_outside_tmp_dir", lambda: os.listdir(os.path.dirname(OUTSIDE_TMP)))83attempt("spawn_curl", curl)84attempt("spawn_ls", lambda: subprocess.run(["/bin/ls", "/"], capture_output=True, check=True))85attempt("spawn_shell", lambda: os.system("/bin/echo hi > /dev/null") == 0 or (_ for _ in ()).throw(RuntimeError("shell failed")))86attempt("spawn_self", spawn_self)87attempt("signal_other_process", lambda: os.kill(VICTIM, signal.SIGTERM))88attempt("read_system_lib", lambda: open("/usr/share/dict/words").read(10))89attempt("granted_read", lambda: open(os.path.join(GRANT_R, "note.txt")).read())90attempt("granted_read_write_refused", write(os.path.join(GRANT_R, "new.txt")))91attempt("granted_write", write(os.path.join(GRANT_W, "result.txt")))92attempt("spawn_osascript", lambda: subprocess.run(["/usr/bin/osascript", "-e", "return 1"], check=True, capture_output=True))93print(json.dumps(out))94"""9596@Suite(.serialized) struct RestrictedModeTests {97    struct Probe { var result: PythonRunResult; var seen: [String: String]; var victimAlive: Bool; var desktopWritten: Bool; var outsideWritten: Bool; var connections: Int; var grantedWrite: Bool }9899    func probe(restricted: Bool, on py: PythonInterpreter, allowNetwork: Bool = false) async throws -> Probe {100        let fm = FileManager.default101        let listener = try LocalListener()102        let id = UUID().uuidString103        let outside = "/tmp/orix-outside-\(id)/file.txt"104        try fm.createDirectory(atPath: "/tmp/orix-outside-\(id)", withIntermediateDirectories: true)105        let desktop = NSHomeDirectory() + "/Desktop/.orix-sheet-python-test-\(id).txt"106        let grantR = fm.temporaryDirectory.appendingPathComponent("orix-grant-r-\(id)"), grantW = fm.temporaryDirectory.appendingPathComponent("orix-grant-w-\(id)")107        try fm.createDirectory(at: grantR, withIntermediateDirectories: true); try fm.createDirectory(at: grantW, withIntermediateDirectories: true)108        try Data("granted".utf8).write(to: grantR.appendingPathComponent("note.txt"))109        let victim = Process(); victim.executableURL = URL(fileURLWithPath: "/bin/sleep"); victim.arguments = ["60"]; try victim.run()110        defer {111            victim.terminate()112            for p in ["/tmp/orix-outside-\(id)", desktop, grantR.path, grantW.path] { try? fm.removeItem(atPath: p) }113        }114        func lit(_ s: String) -> String { "\"" + s.replacingOccurrences(of: "\\", with: "\\\\").replacingOccurrences(of: "\"", with: "\\\"") + "\"" }115        var code = probeScript116        for (k, v) in [("@PORT@", String(listener.port)), ("@OUTSIDE@", lit(outside)), ("@DESKTOP@", lit(desktop)), ("@VICTIM@", String(victim.processIdentifier)),117                       ("@GR@", lit(grantR.path)), ("@GW@", lit(grantW.path))] { code = code.replacingOccurrences(of: k, with: v) }118        let r = try await runPython(code, on: py, restricted: restricted, configure: {119            $0.readablePaths = [grantR]; $0.writablePaths = [grantW]; $0.allowNetwork = allowNetwork120        })121        try await Task.sleep(nanoseconds: 300_000_000)122        return Probe(result: r, seen: (report(r) as? [String: String]) ?? [:], victimAlive: victim.isRunning, desktopWritten: fm.fileExists(atPath: desktop),123                     outsideWritten: fm.fileExists(atPath: outside), connections: listener.connections, grantedWrite: fm.fileExists(atPath: grantW.appendingPathComponent("result.txt").path))124    }125126    static let denied = ["tcp_connect", "udp_send", "tcp_listen", "write_desktop", "write_outside_tmp", "read_ssh", "read_home_listing", "read_keychains",127                         "read_documents", "read_app_support", "read_volumes", "read_outside_tmp_dir", "spawn_curl", "spawn_ls", "spawn_shell", "signal_other_process",128                         "granted_read_write_refused", "spawn_osascript"]129    static let allowed = ["write_run_dir", "write_cwd", "spawn_self", "read_system_lib", "granted_read", "granted_write"]130131    @Test(.enabled(Interp.noPython) { await Interp.any != nil })132    func restrictedModeDeniesWhatItClaims() async throws {133        let py = try #require(await Interp.any)134        try #require(await PythonSandbox.unavailabilityReason() == nil, "sandbox-exec does not work on this machine")135        let p = try await probe(restricted: true, on: py)136        #expect(p.result.succeeded, "\(p.result.stderr)")137        print("RESTRICTED MODE OBSERVATIONS (\(py.path), macOS \(ProcessInfo.processInfo.operatingSystemVersionString)):")138        for k in p.seen.keys.sorted() { print("  \(k): \(p.seen[k] ?? "")") }139        for k in Self.denied where FileManager.default.fileExists(atPath: NSHomeDirectory() + "/.ssh") || k != "read_ssh" {140            #expect(p.seen[k] != nil && p.seen[k] != "ok", "\(k) should be denied in restricted mode, got \(p.seen[k] ?? "nil")")141        }142        for k in Self.allowed { #expect(p.seen[k] == "ok", "\(k) should work in restricted mode, got \(p.seen[k] ?? "nil")") }143        // independent evidence, not just what the script says144        #expect(p.connections == 0, "no connection may reach the listener")145        #expect(!p.desktopWritten && !p.outsideWritten && p.grantedWrite)146        #expect(p.victimAlive, "a restricted script must not be able to signal other processes")147    }148149    @Test(.enabled(Interp.noPython) { await Interp.any != nil })150    func unrestrictedModeAllowsTheSameThings() async throws {151        let py = try #require(await Interp.any)152        let p = try await probe(restricted: false, on: py)153        #expect(p.result.succeeded, "\(p.result.stderr)")154        for k in Self.denied where k != "granted_read_write_refused" && (FileManager.default.fileExists(atPath: NSHomeDirectory() + "/.ssh") || k != "read_ssh") {155            #expect(p.seen[k] == "ok", "\(k) should succeed without restricted mode (this proves the restricted test measures the sandbox), got \(p.seen[k] ?? "nil")")156        }157        #expect(p.connections >= 2 && p.desktopWritten && p.outsideWritten && !p.victimAlive)158    }159160    @Test(.enabled(Interp.noPython) { await Interp.any != nil })161    func networkCanBeGrantedExplicitly() async throws {162        let py = try #require(await Interp.any)163        let p = try await probe(restricted: true, on: py, allowNetwork: true)164        #expect(p.seen["tcp_connect"] == "ok" && p.connections >= 1)165        #expect(p.seen["write_desktop"] != "ok" && p.seen["read_ssh"] != "ok" && p.seen["spawn_curl"] != "ok", "only the network is opened")166    }167168    @Test(.enabled(Interp.noPandas) { await Interp.scientific != nil })169    func scientificStackWorksRestricted() async throws {170        let py = try #require(await Interp.scientific)171        var code = "import numpy as np, pandas as pd, matplotlib.pyplot as plt\n"172        for (pkg, stmt) in [("scipy", "import scipy.stats, scipy.linalg; scipy.stats.norm.cdf(0)"), ("statsmodels", "import statsmodels.api as sm; sm.OLS([1.0,2,3.1],[[1,1],[1,2],[1,3]]).fit()"),173                            ("scikit-learn", "from sklearn.linear_model import LinearRegression; LinearRegression().fit([[1],[2]],[1,2])"),174                            ("polars", "import polars as pl; pl.DataFrame({'a':[1]}).select(pl.col('a').sum())"), ("duckdb", "import duckdb; assert duckdb.sql('select 42').fetchall() == [(42,)]"),175                            ("pyarrow", "import pyarrow as pa; pa.table({'a':[1]})"), ("sympy", "import sympy; sympy.integrate(sympy.Symbol('x'))")] where py.has(pkg) {176            code += stmt + "\nprint('\(pkg) ok')\n"177        }178        code += "pd.DataFrame({'a': np.arange(10)}).plot(); print(np.linalg.inv(np.eye(3)).trace())\nimport multiprocessing as mp\nprint(mp.cpu_count() > 0)\n"179        try await runPython(code, on: py, inspect: { r in180            #expect(r.succeeded, "\(r.stderr)")181            #expect(r.figures.count == 1)182            for pkg in ["scipy", "statsmodels", "scikit-learn", "polars", "duckdb", "pyarrow", "sympy"] where py.has(pkg) { #expect(r.stdout.contains("\(pkg) ok")) }183        })184    }185186    @Test(.enabled(Interp.noPython) { await Interp.any != nil })187    func restrictedModeRefusesToRunWithoutTheSandbox() async throws {188        let py = try #require(await Interp.any)189        Env.prepare()190        let marker = FileManager.default.temporaryDirectory.appendingPathComponent("orix-must-not-exist-\(UUID().uuidString)")191        let code = "open(\(String(reflecting: marker.path)), 'w').write('ran')"192        let before = PythonRunner.existingRunDirectories().count193        try await PythonSandbox.$launcherOverride.withValue("/usr/bin/sandbox-exec-removed-in-a-future-macos") {194            do {195                _ = try await PythonRunner(interpreter: py).run(code: code, host: FakeHost())196                Issue.record("the run should have been refused")197            } catch let e as PythonRunError {198                guard case .sandboxUnavailable(let why) = e else { Issue.record("wrong error \(e)"); return }199                #expect(why.contains("does not exist"))200                #expect(e.localizedDescription.contains("was not run"))201            }202        }203        #expect(!FileManager.default.fileExists(atPath: marker.path), "the script must not have run unrestricted")204        #expect(PythonRunner.existingRunDirectories().count <= before + 4)205        let entry = PythonAuditLog.recent(limit: 5_000).first { $0.codeHash == PythonHash.sha256(code) }206        #expect(entry?.outcome == .refused && entry?.restricted == true)207        // a launcher that exists but cannot apply profiles is refused too208        await PythonSandbox.$launcherOverride.withValue("/usr/bin/false") {209            await #expect(throws: PythonRunError.self) { _ = try await PythonRunner(interpreter: py).run(code: code, host: FakeHost()) }210        }211        #expect(!FileManager.default.fileExists(atPath: marker.path))212        // and the unrestricted mode still works when explicitly chosen213        var o = PythonRunOptions(); o.restricted = false214        #expect(try await PythonRunner(interpreter: py).run(code: code, host: FakeHost(), options: o).succeeded)215        try FileManager.default.removeItem(at: marker)216    }217218    @Test func profileEscapesPathsAndNamesTheRunDirectory() {219        let i = PythonInterpreter(path: "/opt/we\"ird/bin/python3", executable: "/opt/we\"ird/bin/python3", version: "3.12.0", prefix: "/opt/we\"ird")220        let profile = PythonSandbox.profile(interpreter: i, runDirectory: URL(fileURLWithPath: "/private/var/folders/aa/bb/T/orix-python-X"), options: .init())221        #expect(profile.contains(#"(literal "/opt/we\"ird/bin/python3")"#))222        #expect(profile.contains("(deny network*)") && profile.contains(#"(subpath "/Users")"#) && profile.contains(#"(allow file-write* (subpath "/private/var/folders/aa/bb/T/orix-python-X")"#))223        var o = PythonRunOptions(); o.allowNetwork = true224        #expect(!PythonSandbox.profile(interpreter: i, runDirectory: URL(fileURLWithPath: "/tmp/x"), options: o).contains("(deny network*)"))225        #expect(PythonSandbox.canonical("/tmp/a") == "/private/tmp/a" && PythonSandbox.canonical("/var/folders/x") == "/private/var/folders/x")226    }227228    // MARK: Resource limits229230    @Test(.enabled(Interp.noPython) { await Interp.any != nil }, arguments: [true, false])231    func cpuLimitIsEnforcedByTheKernel(restricted: Bool) async throws {232        let py = try #require(await Interp.any)233        let t0 = Date()234        let r = try await runPython("import resource\nprint(resource.getrlimit(resource.RLIMIT_CPU), flush=True)\nwhile True: pass", on: py, restricted: restricted,235                                    configure: { $0.cpuSeconds = 1; $0.timeout = 30 })236        #expect(r.cpuLimitExceeded && r.terminationSignal == SIGXCPU && !r.succeeded && !r.timedOut)237        #expect(r.error?.type == "CPULimitExceeded")238        #expect(r.stdout == "(1, 2)\n")239        #expect(Date().timeIntervalSince(t0) < 10)240    }241242    @Test(.enabled(Interp.noPython) { await Interp.any != nil })243    func fileSizeAndOpenFileLimits() async throws {244        let py = try #require(await Interp.any)245        let r = try await runPython("""246        import os, resource, json, tempfile247        out = {"limits": [list(resource.getrlimit(getattr(resource, n))) for n in ("RLIMIT_FSIZE", "RLIMIT_NOFILE", "RLIMIT_CORE")]}248        try:249            fd = os.open(os.path.join(tempfile.gettempdir(), "big.bin"), os.O_WRONLY | os.O_CREAT)250            for _ in range(100): os.write(fd, b"x" * 4096)251            out["fsize"] = "not enforced"252        except OSError as e: out["fsize"] = "enforced: %s (file is %d bytes)" % (e.strerror, os.fstat(fd).st_size)253        try:254            files = [open("/dev/null") for _ in range(100)]; out["nofile"] = "not enforced"255        except OSError as e: out["nofile"] = "enforced: %s" % e.strerror256        files = None257        for name in ("RLIMIT_FSIZE", "RLIMIT_NOFILE", "RLIMIT_CPU"):258            try:259                resource.setrlimit(getattr(resource, name), (resource.RLIM_INFINITY, resource.RLIM_INFINITY)); out["raise_" + name] = "raised"260            except (ValueError, OSError) as e: out["raise_" + name] = "refused"261        print(json.dumps(out))262        """, on: py, configure: { $0.maxFileBytes = 100_000; $0.maxOpenFiles = 64; $0.cpuSeconds = 120 })263        #expect(r.succeeded, "\(r.stderr)")264        let o = report(r)265        #expect((o["fsize"] as? String)?.hasPrefix("enforced") == true && (o["nofile"] as? String)?.hasPrefix("enforced") == true, "\(o)")266        #expect(o["raise_RLIMIT_FSIZE"] as? String == "refused" && o["raise_RLIMIT_NOFILE"] as? String == "refused" && o["raise_RLIMIT_CPU"] as? String == "refused",267                "hard limits cannot be raised again by the script")268        #expect(r.appliedLimits == ["RLIMIT_CPU": 120, "RLIMIT_FSIZE": 100_000, "RLIMIT_NOFILE": 64])269    }270271    @Test(.enabled(Interp.noPython) { await Interp.any != nil })272    func addressSpaceLimitIsNotEnforceableButTheWatchdogIs() async throws {273        let py = try #require(await Interp.any)274        // 1. what the kernel does with RLIMIT_AS / RLIMIT_DATA / RLIMIT_RSS on this macOS275        let k = try await runPython("""276        import resource, json277        out = {}278        for name in ("RLIMIT_AS", "RLIMIT_DATA", "RLIMIT_RSS"):279            try:280                resource.setrlimit(getattr(resource, name), (256 << 20, 256 << 20)); out[name] = "set"281            except (ValueError, OSError) as e: out[name] = "cannot be set: %s" % e282        try:283            b = bytearray(600 << 20); b[::4096] = b"x" * len(b[::4096]); out["allocate_600MB"] = "ok"284        except MemoryError: out["allocate_600MB"] = "MemoryError"285        print(json.dumps(out))286        """, on: py, configure: { $0.maxMemoryBytes = nil })287        let o = report(k)288        print("MEMORY RLIMITS ON THIS macOS: \(o)")289        #expect(o["allocate_600MB"] as? String == "ok", "if this fails, macOS started enforcing a memory rlimit: update SECURITY.md (\(o))")290        // 2. the host watchdog does stop a script whose resident memory exceeds the cap291        let t0 = Date()292        let w = try await runPython("""293        import time294        blocks = []295        for i in range(200):296            b = bytearray(32 << 20); b[::4096] = b"x" * len(b[::4096]); blocks.append(b)297            time.sleep(0.02)298        print("allocated everything")299        """, on: py, configure: { $0.maxMemoryBytes = 400 << 20; $0.timeout = 60 })300        #expect(w.memoryLimitExceeded && !w.succeeded && !w.stdout.contains("allocated everything"))301        #expect(Date().timeIntervalSince(t0) < 20)302        #expect(auditEntry(w)?.outcome == .killed)303    }304}305
Files main
Menu

Theme