SPB Git

Repository Public

ExecutionTests.swift 22.2 KB 395 lines swift

1import Foundation2import Testing3@testable import SheetPython45func alive(_ pid: Int) -> Bool { kill(pid_t(pid), 0) == 0 || errno == EPERM }67/// Pids announced by a script on stdout as "PID <n>" lines.8func announcedPids(_ text: String) -> [Int] {9    text.split(separator: "\n").compactMap { $0.hasPrefix("PID ") ? Int($0.dropFirst(4).trimmingCharacters(in: .whitespaces)) : nil }10}1112func waitUntilDead(_ pids: [Int], seconds: Double = 3) async -> [Int] {13    let end = Date().addingTimeInterval(seconds)14    while Date() < end, pids.contains(where: alive) { try? await Task.sleep(nanoseconds: 50_000_000) }15    return pids.filter(alive)16}1718@Suite(.serialized) struct ExecutionTests {19    @Test(.enabled(Interp.noPython) { await Interp.any != nil })20    func syntaxErrorReportsLine() async throws {21        let py = try #require(await Interp.any)22        let host = FakeHost()23        let r = try await runPython("import orix\nx = 1\norix.active_sheet().range('A1').write(1)\ndef f(:\n    pass\n", on: py, host: host)24        #expect(!r.succeeded && r.exitStatus == 1)25        #expect(r.error?.type == "SyntaxError" && r.error?.line == 4)26        #expect(r.error?.traceback.contains("script.py") == true && r.error?.traceback.contains("runner.py") == false)27        #expect(r.writes.isEmpty, "nothing runs when the script does not compile")28        #expect(r.stderr.contains("SyntaxError"))29    }3031    @Test(.enabled(Interp.noPython) { await Interp.any != nil })32    func runtimeErrorReportsInnermostUserLine() async throws {33        let py = try #require(await Interp.any)34        let r = try await runPython("""35        import json36        def inner(d):37            return d["missing"]      # line 33839        def outer():40            return inner({})         # line 64142        print("before")43        outer()                      # line 944        print("after")45        """, on: py, configure: { $0.scriptName = "my analysis.py" })46        #expect(r.exitStatus == 1 && r.stdout == "before\n")47        let e = try #require(r.error)48        #expect(e.type == "KeyError" && e.line == 3 && e.message.contains("missing"))49        #expect(e.traceback.contains("File \"my analysis.py\", line 9") && e.traceback.contains("line 6, in outer") && e.traceback.contains("return d[\"missing\"]"))50        #expect(!e.traceback.contains("runner.py"))51    }5253    @Test(.enabled(Interp.noPython) { await Interp.any != nil })54    func errorInsideLibraryPointsAtUserLine() async throws {55        let py = try #require(await Interp.any)56        let r = try await runPython("import json\n\n\njson.loads('{bad')\n", on: py)57        #expect(r.error?.type == "JSONDecodeError" && r.error?.line == 4)58    }5960    @Test(.enabled(Interp.noPython) { await Interp.any != nil })61    func systemExitAndWritesOnFailure() async throws {62        let py = try #require(await Interp.any)63        let ok = try await runPython("import sys\nprint('bye')\nsys.exit(0)", on: py)64        #expect(ok.succeeded && ok.error == nil)65        let three = try await runPython("import sys, orix\norix.active_sheet().range('A1').write(1)\nsys.exit(3)", on: py)66        #expect(three.exitStatus == 3 && !three.succeeded && three.error == nil && three.writes.count == 1, "writes are reported but the run is not successful")67        let msg = try await runPython("raise SystemExit('fatal: stop')", on: py)68        #expect(msg.exitStatus == 1 && msg.stderr.contains("fatal: stop"))69    }7071    @Test(.enabled(Interp.noPython) { await Interp.any != nil })72    func timeoutKillsTheScript() async throws {73        let py = try #require(await Interp.any)74        let t0 = Date()75        let r = try await runPython("import os, time\nprint('PID', os.getpid(), flush=True)\nwhile True: time.sleep(0.05)", on: py, configure: { $0.timeout = 1 })76        #expect(r.timedOut && !r.succeeded && !r.cancelled)77        #expect(Date().timeIntervalSince(t0) < 4 && r.duration >= 0.9)78        #expect(await waitUntilDead(announcedPids(r.stdout)).isEmpty)79        #expect(auditEntry(r)?.outcome == .timeout)80    }8182    /// Cancels `code` once it has printed READY and checks that every announced pid is gone.83    func cancelCase(_ code: String, restricted: Bool = true, grace: TimeInterval = 0.5, expectSIGKILL: Bool = false) async throws {84        let py = try #require(await Interp.any)85        Env.prepare()86        let runner = PythonRunner(interpreter: py)87        let box = OutputBox()88        var o = PythonRunOptions(); o.restricted = restricted; o.timeout = 30; o.terminationGrace = grace89        let options = o90        let task = Task { try await runner.run(code: code, host: FakeHost(), options: options, onOutput: { box.add($0, $1) }) }91        let deadline = Date().addingTimeInterval(15)92        while !box.stdout.contains("READY"), Date() < deadline { try await Task.sleep(nanoseconds: 20_000_000) }93        try #require(box.stdout.contains("READY"), "script did not start: \(box.stderr)")94        #expect(runner.isRunning)95        let pids = announcedPids(box.stdout)96        #expect(!pids.isEmpty && pids.allSatisfy(alive))97        let t0 = Date()98        runner.cancel()99        let r = try await task.value100        let took = Date().timeIntervalSince(t0)101        #expect(r.cancelled && !r.succeeded && !r.timedOut)102        #expect(took < grace + 2.5, "cancel took \(took) s")103        if expectSIGKILL { #expect(r.terminationSignal == SIGKILL) }104        #expect(await waitUntilDead(pids).isEmpty, "orphan processes left: \(pids.filter(alive))")105        #expect(r.runDirectory == nil && !runner.isRunning)106        #expect(auditEntry(r)?.outcome == .cancelled)107    }108109    @Test(.enabled(Interp.noPython) { await Interp.any != nil })110    func cancelInfiniteLoop() async throws {111        try await cancelCase("import os\nprint('PID', os.getpid()); print('READY', flush=True)\nwhile True: pass")112    }113114    @Test(.enabled(Interp.noPython) { await Interp.any != nil })115    func cancelSleeping() async throws {116        try await cancelCase("import os, time\nprint('PID', os.getpid()); print('READY', flush=True)\ntime.sleep(600)")117    }118119    @Test(.enabled(Interp.noPandas) { await Interp.scientific != nil })120    func cancelCPUBoundNativeCode() async throws {121        try await cancelCase("import os, numpy as np\nprint('PID', os.getpid()); print('READY', flush=True)\na = np.random.rand(3000, 3000)\nwhile True: a = a @ a; a /= a.max()")122    }123124    @Test(.enabled(Interp.noPython) { await Interp.any != nil })125    func cancelIgnoringSIGTERMGetsSIGKILL() async throws {126        try await cancelCase("import os, signal, time\nsignal.signal(signal.SIGTERM, signal.SIG_IGN)\nprint('PID', os.getpid()); print('READY', flush=True)\nwhile True: time.sleep(0.1)", grace: 0.3, expectSIGKILL: true)127    }128129    @Test(.enabled(Interp.noPython) { await Interp.any != nil }, arguments: [true, false])130    func cancelWithChildProcesses(restricted: Bool) async throws {131        // children: same group, a grandchild, one that left the process group (setsid) and ignores SIGTERM132        try await cancelCase("""133        import os, subprocess, sys, time134        print('PID', os.getpid())135        kids = [subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(600)']) for _ in range(2)]136        kids.append(subprocess.Popen([sys.executable, '-c', 'import subprocess, sys, time; p = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(600)"]); print("PID", p.pid, flush=True); time.sleep(600)']))137        kids.append(subprocess.Popen([sys.executable, '-c', 'import signal, time; signal.signal(signal.SIGTERM, signal.SIG_IGN); time.sleep(600)'], start_new_session=True))138        for k in kids: print('PID', k.pid)139        time.sleep(1.0)140        print('READY', flush=True)141        time.sleep(600)142        """, restricted: restricted)143    }144145    @Test(.enabled(Interp.noPython) { await Interp.any != nil })146    func childrenLeftBehindByAFinishedScriptAreKilled() async throws {147        let py = try #require(await Interp.any)148        let r = try await runPython("""149        import subprocess, sys, time150        p = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(600)'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)151        q = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(600)'])          # keeps our stdout pipe open152        print('PID', p.pid); print('PID', q.pid)153        time.sleep(0.5)154        """, on: py)155        #expect(r.succeeded, "\(r.stderr)")156        let pids = announcedPids(r.stdout)157        #expect(pids.count == 2)158        #expect(await waitUntilDead(pids).isEmpty)159        #expect(r.duration < 5)160    }161162    @Test(.enabled(Interp.noPython) { await Interp.any != nil })163    func taskCancellationStopsTheScript() async throws {164        let py = try #require(await Interp.any)165        Env.prepare()166        let runner = PythonRunner(interpreter: py)167        let box = OutputBox()168        let task = Task { try await runner.run(code: "import os\nprint('PID', os.getpid(), flush=True)\nwhile True: pass", host: FakeHost(), onOutput: { box.add($0, $1) }) }169        let deadline = Date().addingTimeInterval(15)170        while announcedPids(box.stdout).isEmpty, Date() < deadline { try await Task.sleep(nanoseconds: 20_000_000) }171        task.cancel()172        let r = try await task.value173        #expect(r.cancelled)174        #expect(await waitUntilDead(announcedPids(box.stdout)).isEmpty)175    }176177    @Test(.enabled(Interp.noPython) { await Interp.any != nil })178    func oneRunAtATimePerRunner() async throws {179        let py = try #require(await Interp.any)180        Env.prepare()181        let runner = PythonRunner(interpreter: py)182        let first = Task { try await runner.run(code: "import time\ntime.sleep(1.5)", host: FakeHost()) }183        try await Task.sleep(nanoseconds: 400_000_000)184        await #expect(throws: PythonRunError.alreadyRunning) { try await runner.run(code: "print(1)", host: FakeHost()) }185        runner.cancel()186        _ = try await first.value187        let again = try await runner.run(code: "print(2)", host: FakeHost())188        #expect(again.succeeded && again.stdout == "2\n")189    }190191    @Test(.enabled(Interp.noPython) { await Interp.any != nil })192    func outputIsCappedAndStreamed() async throws {193        let py = try #require(await Interp.any)194        let box = OutputBox()195        let r = try await runPython("""196        import sys197        for i in range(200000):198            sys.stdout.write("x" * 99 + "\\n")199        sys.stderr.write("end\\n")200        print("done")201        """, on: py, configure: { $0.maxOutputBytes = 100_000 }, output: box)202        #expect(r.succeeded, "the script still finishes: the rest of its output is drained and dropped")203        #expect(r.outputTruncated)204        #expect(r.stdout.utf8.count + r.stderr.utf8.count == 100_000)205        #expect(box.stderr.contains("[output truncated: more than 100000 bytes]"))206        #expect(box.stdout == r.stdout && box.count > 2)207        let small = try await runPython("print('hi')", on: py)208        #expect(!small.outputTruncated)209    }210211    @Test(.enabled(Interp.noPython) { await Interp.any != nil })212    func streamingArrivesBeforeTheEnd() async throws {213        let py = try #require(await Interp.any)214        Env.prepare()215        let box = OutputBox()216        let runner = PythonRunner(interpreter: py)217        let task = Task { try await runner.run(code: "import time\nprint('early')\ntime.sleep(2)\nprint('late')", host: FakeHost(), onOutput: { box.add($0, $1) }) }218        let deadline = Date().addingTimeInterval(1.5)219        while Date() < deadline, !box.stdout.contains("early") { try await Task.sleep(nanoseconds: 20_000_000) }220        #expect(box.stdout == "early\n", "unbuffered output is delivered while the script is still running")221        #expect(try await task.value.stdout == "early\nlate\n")222    }223224    // MARK: RPC robustness225226    @Test(.enabled(Interp.noPython) { await Interp.any != nil })227    func malformedMessagesAreIgnored() async throws {228        let py = try #require(await Interp.any)229        let r = try await runPython("""230        import os, orix231        os.write(4, b"this is not json\\n")232        os.read(3, 65536)                       # the refusal233        os.write(4, b'{"op": "format_disk", "id": 99}\\n')234        os.read(3, 65536)235        os.write(4, b'[1, 2, 3]\\n')236        os.read(3, 65536)237        os.write(4, b'{"op": "write", "id": 7, "anchor": "A1", "rows": [[1], [2, 3]]}\\n')      # ragged238        print(os.read(3, 65536).decode().strip())239        os.write(4, b'{"op": "write", "id": 8, "anchor": "ZZZZ1", "rows": [[1]]}\\n')240        print(os.read(3, 65536).decode().strip())241        os.write(4, b'{"op": "figure", "id": 9, "path": "/etc/passwd"}\\n')242        print(os.read(3, 65536).decode().strip())243        print(orix.active_sheet().name)         # the channel still works244        orix.active_sheet().range("B2").write("ok")245        """, on: py)246        #expect(r.succeeded, "\(r.stderr)")247        #expect(r.warnings.count == 3)248        #expect(r.stdout.contains("rows have different lengths") && r.stdout.contains("invalid anchor") && r.stdout.contains("not a file of this run"))249        #expect(r.stdout.hasSuffix("Data\n"))250        #expect(r.writes.count == 1 && r.figures.isEmpty)251    }252253    @Test(.enabled(Interp.noPython) { await Interp.any != nil })254    func scriptClosingThePipes() async throws {255        let py = try #require(await Interp.any)256        let r = try await runPython("""257        import os, orix258        orix.active_sheet().range("A1").write(1)259        os.close(4); os.close(3)260        try:261            orix.active_sheet()262        except orix.OrixError as e:263            print("closed:", e)264        print("still running")265        """, on: py)266        #expect(r.exitStatus == 0 && r.succeeded, "\(r.stderr)")267        #expect(r.stdout.contains("closed:") && r.stdout.hasSuffix("still running\n"))268        #expect(r.writes.count == 1)269        let failing = try await runPython("import os\nos.close(4)\nraise ValueError('after close')", on: py)270        #expect(failing.exitStatus == 1 && !failing.succeeded && failing.error == nil && failing.stderr.contains("ValueError: after close"))271    }272273    @Test(.enabled(Interp.noPython) { await Interp.any != nil })274    func hostErrorsReachTheScript() async throws {275        let py = try #require(await Interp.any)276        let host = FakeHost(); host.failReads = true277        let caught = try await runPython("""278        import orix279        try:280            orix.active_sheet().range("A1:B2").values281        except orix.OrixError as e:282            print("caught:", e)283        """, on: py, host: host)284        #expect(caught.succeeded && caught.stdout == "caught: the host refused this read\n")285        let uncaught = try await runPython("import orix\n\norix.active_sheet().range('A1').value\n", on: py, host: host)286        #expect(uncaught.error?.type == "OrixError" && uncaught.error?.line == 3 && uncaught.error?.message == "the host refused this read")287    }288289    @Test(.enabled(Interp.noPython) { await Interp.any != nil })290    func killedWhileHostIsAnswering() async throws {291        let py = try #require(await Interp.any)292        let host = FakeHost(); host.readDelay = 3293        let t0 = Date()294        let r = try await runPython("import orix\norix.active_sheet().range('A1').value", on: py, host: host, configure: { $0.timeout = 0.5 })295        #expect(r.timedOut && Date().timeIntervalSince(t0) < 6)296    }297298    @Test(.enabled(Interp.noPython) { await Interp.any != nil })299    func scriptSpammingTheChannelAndExiting() async throws {300        let py = try #require(await Interp.any)301        let r = try await runPython("""302        import os303        for i in range(2000):304            os.write(4, b'{"op":"active_sheet","id":%d}\\n' % i)      # never reads the answers305        os._exit(0)306        """, on: py)307        #expect(r.exitStatus == 0 && r.duration < 10)308    }309310    @Test(.enabled(Interp.noPython) { await Interp.any != nil })311    func readAfterWriteSeesThePreRunSheet() async throws {312        let py = try #require(await Interp.any)313        let host = FakeHost([.init(name: "S", rows: [[.number(1)]])])314        let r = try await runPython("import orix\ns = orix.active_sheet()\ns.range('A1').write(99)\nprint(s.range('A1').value)", on: py, host: host)315        #expect(r.stdout == "1.0\n" && r.writes.count == 1, "writes are buffered: documented behaviour")316    }317318    @Test(.enabled(Interp.noPython) { await Interp.any != nil })319    func environmentIsAnAllowList() async throws {320        let py = try #require(await Interp.any)321        setenv("ORIX_TEST_SECRET", "hunter2", 1); setenv("PYTHONSTARTUP", "/tmp/nope.py", 1); setenv("PYTHONPATH", "/tmp/evil", 1)322        defer { unsetenv("ORIX_TEST_SECRET"); unsetenv("PYTHONSTARTUP"); unsetenv("PYTHONPATH") }323        let r = try await runPython("""324        import os, sys, json, tempfile325        print(json.dumps({"env": sorted(os.environ), "home": os.environ["HOME"], "cwd": os.getcwd(), "tmp": tempfile.gettempdir(),326                          "path": sys.path, "flags": [sys.flags.isolated, sys.flags.no_user_site, sys.flags.dont_write_bytecode, sys.flags.ignore_environment],327                          "stdin": sys.stdin.read(), "argv": sys.argv, "name": __name__, "restricted": __import__("orix").RESTRICTED,328                          "fds": sorted(int(f) for f in os.listdir("/dev/fd") if int(f) < 64)}))329        """, on: py, configure: { $0.extraEnvironment = ["MY_SETTING": "1", "PYTHONINSPECT": "1", "DYLD_INSERT_LIBRARIES": "/tmp/x.dylib"] })330        #expect(r.succeeded, "\(r.stderr)")331        let o = report(r)332        let env = Set(try #require(o["env"] as? [String]))333        #expect(env.isSubset(of: ["HOME", "LANG", "LC_ALL", "LC_CTYPE", "TZ", "PATH", "TMPDIR", "XDG_CACHE_HOME", "MPLBACKEND", "MPLCONFIGDIR", "ORIX_SHEET", "MY_SETTING", "__CF_USER_TEXT_ENCODING"]), "\(env)")334        #expect(env.contains("MY_SETTING") && !env.contains("ORIX_TEST_SECRET"))335        let home = try #require(o["home"] as? String), cwd = try #require(o["cwd"] as? String)336        #expect(home.contains(PythonPaths.runDirectoryPrefix) && cwd == home && (o["tmp"] as? String)?.contains(PythonPaths.runDirectoryPrefix) == true)337        #expect((o["path"] as? [String])?.contains { $0.hasPrefix("/tmp/evil") || $0.contains("orix-python-") } == false)338        #expect(o["flags"] as? [Int] == [1, 1, 1, 1])339        #expect(o["stdin"] as? String == "" && o["argv"] as? [String] == ["script.py"] && o["name"] as? String == "__main__" && o["restricted"] as? Bool == true)340        let fds = try #require(o["fds"] as? [Int])341        #expect(Set(fds).subtracting([0, 1, 2, 3, 4]).count <= 1, "only stdio and the two RPC descriptors are inherited (plus the one listing /dev/fd): \(fds)")342        #expect(!FileManager.default.fileExists(atPath: home))343    }344}345346@Suite(.serialized) struct AppExitTests {347    @Test(.enabled(Interp.noPython) { await Interp.any != nil })348    func quittingTheAppKillsRunningScriptsAndRemovesTheirDirectories() async throws {349        let py = try #require(await Interp.any)350        Env.prepare()351        let box = OutputBox()352        let runner = PythonRunner(interpreter: py)353        let task = Task { try await runner.run(code: "import os, subprocess, sys, time\nk = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(600)'])\nprint('PID', os.getpid()); print('PID', k.pid); print('DIR', os.getcwd(), flush=True)\ntime.sleep(600)",354                                                host: FakeHost(), onOutput: { box.add($0, $1) }) }355        let deadline = Date().addingTimeInterval(15)356        while !box.stdout.contains("DIR "), Date() < deadline { try await Task.sleep(nanoseconds: 20_000_000) }357        let pids = announcedPids(box.stdout)358        let cwd = try #require(box.stdout.split(separator: "\n").first { $0.hasPrefix("DIR ") }.map { String($0.dropFirst(4)) })359        let runDirectory = URL(fileURLWithPath: cwd).deletingLastPathComponent()360        #expect(pids.count == 2 && pids.allSatisfy(alive) && FileManager.default.fileExists(atPath: runDirectory.path))361        #expect(PythonRunner.runningScriptCount >= 1)362        // what the app does in applicationWillTerminate (restricted to this test's run: other suites run in parallel)363        LiveRuns.shared.killAll { PythonSandbox.canonical($0.path) == PythonSandbox.canonical(runDirectory.path) }364        #expect(await waitUntilDead(pids).isEmpty)365        #expect(!FileManager.default.fileExists(atPath: runDirectory.path))366        let r = try await task.value367        #expect(r.cancelled && r.terminationSignal == SIGKILL)368    }369370    /// The app is killed without any chance to clean up: the script notices it lost its parent and kills its own group.371    @Test(.enabled(Interp.noPython) { await Interp.any != nil }, arguments: [true, false])372    func scriptDoesNotOutliveACrashedApp(restricted: Bool) async throws {373        let py = try #require(await Interp.any)374        Env.prepare()375        // a stand-in "app": a shell that starts the runner exactly as PythonRunner does, then is SIGKILLed376        let prepared = try PreparedRun(interpreter: py, code: "import os, subprocess, sys, time\nk = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(600)'])\nopen('pids', 'w').write('%d %d' % (os.getpid(), k.pid))\ntime.sleep(600)",377                                       options: { var o = PythonRunOptions(); o.restricted = restricted; return o }())378        defer { prepared.removeAll() }379        func quote(_ s: String) -> String { "'" + s.replacingOccurrences(of: "'", with: "'\\''") + "'" }380        let command = ([prepared.executable] + prepared.arguments).map(quote).joined(separator: " ") + " 3</dev/null 4>/dev/null & wait"381        let app = Process()382        app.executableURL = URL(fileURLWithPath: "/bin/sh"); app.arguments = ["-c", command]383        app.environment = prepared.environment; app.currentDirectoryURL = prepared.workingDirectory384        app.standardOutput = FileHandle.nullDevice; app.standardError = FileHandle.nullDevice385        try app.run()386        let file = prepared.workingDirectory.appendingPathComponent("pids")387        let deadline = Date().addingTimeInterval(15)388        while (try? String(contentsOf: file, encoding: .utf8))?.split(separator: " ").count != 2, Date() < deadline { try await Task.sleep(nanoseconds: 50_000_000) }389        let pids = (try String(contentsOf: file, encoding: .utf8)).split(separator: " ").compactMap { Int($0) }390        try #require(pids.count == 2 && pids.allSatisfy(alive))391        kill(app.processIdentifier, SIGKILL)            // the "app" dies abruptly392        #expect(await waitUntilDead(pids, seconds: 5).isEmpty, "the script and its child must exit on their own: \(pids.filter(alive))")393    }394}395
Files main
Menu

Theme