Repository Public
1import Foundation2import Testing3@testable import SheetPython45func alive(_ pid: Int) -> Bool { kill(pid_t(pid), 0) == 0 || errno == EPERM }67/// Pids announced by a script on stdout as "PID <n>" lines.8func announcedPids(_ text: String) -> [Int] {9 text.split(separator: "\n").compactMap { $0.hasPrefix("PID ") ? Int($0.dropFirst(4).trimmingCharacters(in: .whitespaces)) : nil }10}1112func waitUntilDead(_ pids: [Int], seconds: Double = 3) async -> [Int] {13 let end = Date().addingTimeInterval(seconds)14 while Date() < end, pids.contains(where: alive) { try? await Task.sleep(nanoseconds: 50_000_000) }15 return pids.filter(alive)16}1718@Suite(.serialized) struct ExecutionTests {19 @Test(.enabled(Interp.noPython) { await Interp.any != nil })20 func syntaxErrorReportsLine() async throws {21 let py = try #require(await Interp.any)22 let host = FakeHost()23 let r = try await runPython("import orix\nx = 1\norix.active_sheet().range('A1').write(1)\ndef f(:\n pass\n", on: py, host: host)24 #expect(!r.succeeded && r.exitStatus == 1)25 #expect(r.error?.type == "SyntaxError" && r.error?.line == 4)26 #expect(r.error?.traceback.contains("script.py") == true && r.error?.traceback.contains("runner.py") == false)27 #expect(r.writes.isEmpty, "nothing runs when the script does not compile")28 #expect(r.stderr.contains("SyntaxError"))29 }3031 @Test(.enabled(Interp.noPython) { await Interp.any != nil })32 func runtimeErrorReportsInnermostUserLine() async throws {33 let py = try #require(await Interp.any)34 let r = try await runPython("""35 import json36 def inner(d):37 return d["missing"] # line 33839 def outer():40 return inner({}) # line 64142 print("before")43 outer() # line 944 print("after")45 """, on: py, configure: { $0.scriptName = "my analysis.py" })46 #expect(r.exitStatus == 1 && r.stdout == "before\n")47 let e = try #require(r.error)48 #expect(e.type == "KeyError" && e.line == 3 && e.message.contains("missing"))49 #expect(e.traceback.contains("File \"my analysis.py\", line 9") && e.traceback.contains("line 6, in outer") && e.traceback.contains("return d[\"missing\"]"))50 #expect(!e.traceback.contains("runner.py"))51 }5253 @Test(.enabled(Interp.noPython) { await Interp.any != nil })54 func errorInsideLibraryPointsAtUserLine() async throws {55 let py = try #require(await Interp.any)56 let r = try await runPython("import json\n\n\njson.loads('{bad')\n", on: py)57 #expect(r.error?.type == "JSONDecodeError" && r.error?.line == 4)58 }5960 @Test(.enabled(Interp.noPython) { await Interp.any != nil })61 func systemExitAndWritesOnFailure() async throws {62 let py = try #require(await Interp.any)63 let ok = try await runPython("import sys\nprint('bye')\nsys.exit(0)", on: py)64 #expect(ok.succeeded && ok.error == nil)65 let three = try await runPython("import sys, orix\norix.active_sheet().range('A1').write(1)\nsys.exit(3)", on: py)66 #expect(three.exitStatus == 3 && !three.succeeded && three.error == nil && three.writes.count == 1, "writes are reported but the run is not successful")67 let msg = try await runPython("raise SystemExit('fatal: stop')", on: py)68 #expect(msg.exitStatus == 1 && msg.stderr.contains("fatal: stop"))69 }7071 @Test(.enabled(Interp.noPython) { await Interp.any != nil })72 func timeoutKillsTheScript() async throws {73 let py = try #require(await Interp.any)74 let t0 = Date()75 let r = try await runPython("import os, time\nprint('PID', os.getpid(), flush=True)\nwhile True: time.sleep(0.05)", on: py, configure: { $0.timeout = 1 })76 #expect(r.timedOut && !r.succeeded && !r.cancelled)77 #expect(Date().timeIntervalSince(t0) < 4 && r.duration >= 0.9)78 #expect(await waitUntilDead(announcedPids(r.stdout)).isEmpty)79 #expect(auditEntry(r)?.outcome == .timeout)80 }8182 /// Cancels `code` once it has printed READY and checks that every announced pid is gone.83 func cancelCase(_ code: String, restricted: Bool = true, grace: TimeInterval = 0.5, expectSIGKILL: Bool = false) async throws {84 let py = try #require(await Interp.any)85 Env.prepare()86 let runner = PythonRunner(interpreter: py)87 let box = OutputBox()88 var o = PythonRunOptions(); o.restricted = restricted; o.timeout = 30; o.terminationGrace = grace89 let options = o90 let task = Task { try await runner.run(code: code, host: FakeHost(), options: options, onOutput: { box.add($0, $1) }) }91 let deadline = Date().addingTimeInterval(15)92 while !box.stdout.contains("READY"), Date() < deadline { try await Task.sleep(nanoseconds: 20_000_000) }93 try #require(box.stdout.contains("READY"), "script did not start: \(box.stderr)")94 #expect(runner.isRunning)95 let pids = announcedPids(box.stdout)96 #expect(!pids.isEmpty && pids.allSatisfy(alive))97 let t0 = Date()98 runner.cancel()99 let r = try await task.value100 let took = Date().timeIntervalSince(t0)101 #expect(r.cancelled && !r.succeeded && !r.timedOut)102 #expect(took < grace + 2.5, "cancel took \(took) s")103 if expectSIGKILL { #expect(r.terminationSignal == SIGKILL) }104 #expect(await waitUntilDead(pids).isEmpty, "orphan processes left: \(pids.filter(alive))")105 #expect(r.runDirectory == nil && !runner.isRunning)106 #expect(auditEntry(r)?.outcome == .cancelled)107 }108109 @Test(.enabled(Interp.noPython) { await Interp.any != nil })110 func cancelInfiniteLoop() async throws {111 try await cancelCase("import os\nprint('PID', os.getpid()); print('READY', flush=True)\nwhile True: pass")112 }113114 @Test(.enabled(Interp.noPython) { await Interp.any != nil })115 func cancelSleeping() async throws {116 try await cancelCase("import os, time\nprint('PID', os.getpid()); print('READY', flush=True)\ntime.sleep(600)")117 }118119 @Test(.enabled(Interp.noPandas) { await Interp.scientific != nil })120 func cancelCPUBoundNativeCode() async throws {121 try await cancelCase("import os, numpy as np\nprint('PID', os.getpid()); print('READY', flush=True)\na = np.random.rand(3000, 3000)\nwhile True: a = a @ a; a /= a.max()")122 }123124 @Test(.enabled(Interp.noPython) { await Interp.any != nil })125 func cancelIgnoringSIGTERMGetsSIGKILL() async throws {126 try await cancelCase("import os, signal, time\nsignal.signal(signal.SIGTERM, signal.SIG_IGN)\nprint('PID', os.getpid()); print('READY', flush=True)\nwhile True: time.sleep(0.1)", grace: 0.3, expectSIGKILL: true)127 }128129 @Test(.enabled(Interp.noPython) { await Interp.any != nil }, arguments: [true, false])130 func cancelWithChildProcesses(restricted: Bool) async throws {131 // children: same group, a grandchild, one that left the process group (setsid) and ignores SIGTERM132 try await cancelCase("""133 import os, subprocess, sys, time134 print('PID', os.getpid())135 kids = [subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(600)']) for _ in range(2)]136 kids.append(subprocess.Popen([sys.executable, '-c', 'import subprocess, sys, time; p = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(600)"]); print("PID", p.pid, flush=True); time.sleep(600)']))137 kids.append(subprocess.Popen([sys.executable, '-c', 'import signal, time; signal.signal(signal.SIGTERM, signal.SIG_IGN); time.sleep(600)'], start_new_session=True))138 for k in kids: print('PID', k.pid)139 time.sleep(1.0)140 print('READY', flush=True)141 time.sleep(600)142 """, restricted: restricted)143 }144145 @Test(.enabled(Interp.noPython) { await Interp.any != nil })146 func childrenLeftBehindByAFinishedScriptAreKilled() async throws {147 let py = try #require(await Interp.any)148 let r = try await runPython("""149 import subprocess, sys, time150 p = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(600)'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)151 q = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(600)']) # keeps our stdout pipe open152 print('PID', p.pid); print('PID', q.pid)153 time.sleep(0.5)154 """, on: py)155 #expect(r.succeeded, "\(r.stderr)")156 let pids = announcedPids(r.stdout)157 #expect(pids.count == 2)158 #expect(await waitUntilDead(pids).isEmpty)159 #expect(r.duration < 5)160 }161162 @Test(.enabled(Interp.noPython) { await Interp.any != nil })163 func taskCancellationStopsTheScript() async throws {164 let py = try #require(await Interp.any)165 Env.prepare()166 let runner = PythonRunner(interpreter: py)167 let box = OutputBox()168 let task = Task { try await runner.run(code: "import os\nprint('PID', os.getpid(), flush=True)\nwhile True: pass", host: FakeHost(), onOutput: { box.add($0, $1) }) }169 let deadline = Date().addingTimeInterval(15)170 while announcedPids(box.stdout).isEmpty, Date() < deadline { try await Task.sleep(nanoseconds: 20_000_000) }171 task.cancel()172 let r = try await task.value173 #expect(r.cancelled)174 #expect(await waitUntilDead(announcedPids(box.stdout)).isEmpty)175 }176177 @Test(.enabled(Interp.noPython) { await Interp.any != nil })178 func oneRunAtATimePerRunner() async throws {179 let py = try #require(await Interp.any)180 Env.prepare()181 let runner = PythonRunner(interpreter: py)182 let first = Task { try await runner.run(code: "import time\ntime.sleep(1.5)", host: FakeHost()) }183 try await Task.sleep(nanoseconds: 400_000_000)184 await #expect(throws: PythonRunError.alreadyRunning) { try await runner.run(code: "print(1)", host: FakeHost()) }185 runner.cancel()186 _ = try await first.value187 let again = try await runner.run(code: "print(2)", host: FakeHost())188 #expect(again.succeeded && again.stdout == "2\n")189 }190191 @Test(.enabled(Interp.noPython) { await Interp.any != nil })192 func outputIsCappedAndStreamed() async throws {193 let py = try #require(await Interp.any)194 let box = OutputBox()195 let r = try await runPython("""196 import sys197 for i in range(200000):198 sys.stdout.write("x" * 99 + "\\n")199 sys.stderr.write("end\\n")200 print("done")201 """, on: py, configure: { $0.maxOutputBytes = 100_000 }, output: box)202 #expect(r.succeeded, "the script still finishes: the rest of its output is drained and dropped")203 #expect(r.outputTruncated)204 #expect(r.stdout.utf8.count + r.stderr.utf8.count == 100_000)205 #expect(box.stderr.contains("[output truncated: more than 100000 bytes]"))206 #expect(box.stdout == r.stdout && box.count > 2)207 let small = try await runPython("print('hi')", on: py)208 #expect(!small.outputTruncated)209 }210211 @Test(.enabled(Interp.noPython) { await Interp.any != nil })212 func streamingArrivesBeforeTheEnd() async throws {213 let py = try #require(await Interp.any)214 Env.prepare()215 let box = OutputBox()216 let runner = PythonRunner(interpreter: py)217 let task = Task { try await runner.run(code: "import time\nprint('early')\ntime.sleep(2)\nprint('late')", host: FakeHost(), onOutput: { box.add($0, $1) }) }218 let deadline = Date().addingTimeInterval(1.5)219 while Date() < deadline, !box.stdout.contains("early") { try await Task.sleep(nanoseconds: 20_000_000) }220 #expect(box.stdout == "early\n", "unbuffered output is delivered while the script is still running")221 #expect(try await task.value.stdout == "early\nlate\n")222 }223224 // MARK: RPC robustness225226 @Test(.enabled(Interp.noPython) { await Interp.any != nil })227 func malformedMessagesAreIgnored() async throws {228 let py = try #require(await Interp.any)229 let r = try await runPython("""230 import os, orix231 os.write(4, b"this is not json\\n")232 os.read(3, 65536) # the refusal233 os.write(4, b'{"op": "format_disk", "id": 99}\\n')234 os.read(3, 65536)235 os.write(4, b'[1, 2, 3]\\n')236 os.read(3, 65536)237 os.write(4, b'{"op": "write", "id": 7, "anchor": "A1", "rows": [[1], [2, 3]]}\\n') # ragged238 print(os.read(3, 65536).decode().strip())239 os.write(4, b'{"op": "write", "id": 8, "anchor": "ZZZZ1", "rows": [[1]]}\\n')240 print(os.read(3, 65536).decode().strip())241 os.write(4, b'{"op": "figure", "id": 9, "path": "/etc/passwd"}\\n')242 print(os.read(3, 65536).decode().strip())243 print(orix.active_sheet().name) # the channel still works244 orix.active_sheet().range("B2").write("ok")245 """, on: py)246 #expect(r.succeeded, "\(r.stderr)")247 #expect(r.warnings.count == 3)248 #expect(r.stdout.contains("rows have different lengths") && r.stdout.contains("invalid anchor") && r.stdout.contains("not a file of this run"))249 #expect(r.stdout.hasSuffix("Data\n"))250 #expect(r.writes.count == 1 && r.figures.isEmpty)251 }252253 @Test(.enabled(Interp.noPython) { await Interp.any != nil })254 func scriptClosingThePipes() async throws {255 let py = try #require(await Interp.any)256 let r = try await runPython("""257 import os, orix258 orix.active_sheet().range("A1").write(1)259 os.close(4); os.close(3)260 try:261 orix.active_sheet()262 except orix.OrixError as e:263 print("closed:", e)264 print("still running")265 """, on: py)266 #expect(r.exitStatus == 0 && r.succeeded, "\(r.stderr)")267 #expect(r.stdout.contains("closed:") && r.stdout.hasSuffix("still running\n"))268 #expect(r.writes.count == 1)269 let failing = try await runPython("import os\nos.close(4)\nraise ValueError('after close')", on: py)270 #expect(failing.exitStatus == 1 && !failing.succeeded && failing.error == nil && failing.stderr.contains("ValueError: after close"))271 }272273 @Test(.enabled(Interp.noPython) { await Interp.any != nil })274 func hostErrorsReachTheScript() async throws {275 let py = try #require(await Interp.any)276 let host = FakeHost(); host.failReads = true277 let caught = try await runPython("""278 import orix279 try:280 orix.active_sheet().range("A1:B2").values281 except orix.OrixError as e:282 print("caught:", e)283 """, on: py, host: host)284 #expect(caught.succeeded && caught.stdout == "caught: the host refused this read\n")285 let uncaught = try await runPython("import orix\n\norix.active_sheet().range('A1').value\n", on: py, host: host)286 #expect(uncaught.error?.type == "OrixError" && uncaught.error?.line == 3 && uncaught.error?.message == "the host refused this read")287 }288289 @Test(.enabled(Interp.noPython) { await Interp.any != nil })290 func killedWhileHostIsAnswering() async throws {291 let py = try #require(await Interp.any)292 let host = FakeHost(); host.readDelay = 3293 let t0 = Date()294 let r = try await runPython("import orix\norix.active_sheet().range('A1').value", on: py, host: host, configure: { $0.timeout = 0.5 })295 #expect(r.timedOut && Date().timeIntervalSince(t0) < 6)296 }297298 @Test(.enabled(Interp.noPython) { await Interp.any != nil })299 func scriptSpammingTheChannelAndExiting() async throws {300 let py = try #require(await Interp.any)301 let r = try await runPython("""302 import os303 for i in range(2000):304 os.write(4, b'{"op":"active_sheet","id":%d}\\n' % i) # never reads the answers305 os._exit(0)306 """, on: py)307 #expect(r.exitStatus == 0 && r.duration < 10)308 }309310 @Test(.enabled(Interp.noPython) { await Interp.any != nil })311 func readAfterWriteSeesThePreRunSheet() async throws {312 let py = try #require(await Interp.any)313 let host = FakeHost([.init(name: "S", rows: [[.number(1)]])])314 let r = try await runPython("import orix\ns = orix.active_sheet()\ns.range('A1').write(99)\nprint(s.range('A1').value)", on: py, host: host)315 #expect(r.stdout == "1.0\n" && r.writes.count == 1, "writes are buffered: documented behaviour")316 }317318 @Test(.enabled(Interp.noPython) { await Interp.any != nil })319 func environmentIsAnAllowList() async throws {320 let py = try #require(await Interp.any)321 setenv("ORIX_TEST_SECRET", "hunter2", 1); setenv("PYTHONSTARTUP", "/tmp/nope.py", 1); setenv("PYTHONPATH", "/tmp/evil", 1)322 defer { unsetenv("ORIX_TEST_SECRET"); unsetenv("PYTHONSTARTUP"); unsetenv("PYTHONPATH") }323 let r = try await runPython("""324 import os, sys, json, tempfile325 print(json.dumps({"env": sorted(os.environ), "home": os.environ["HOME"], "cwd": os.getcwd(), "tmp": tempfile.gettempdir(),326 "path": sys.path, "flags": [sys.flags.isolated, sys.flags.no_user_site, sys.flags.dont_write_bytecode, sys.flags.ignore_environment],327 "stdin": sys.stdin.read(), "argv": sys.argv, "name": __name__, "restricted": __import__("orix").RESTRICTED,328 "fds": sorted(int(f) for f in os.listdir("/dev/fd") if int(f) < 64)}))329 """, on: py, configure: { $0.extraEnvironment = ["MY_SETTING": "1", "PYTHONINSPECT": "1", "DYLD_INSERT_LIBRARIES": "/tmp/x.dylib"] })330 #expect(r.succeeded, "\(r.stderr)")331 let o = report(r)332 let env = Set(try #require(o["env"] as? [String]))333 #expect(env.isSubset(of: ["HOME", "LANG", "LC_ALL", "LC_CTYPE", "TZ", "PATH", "TMPDIR", "XDG_CACHE_HOME", "MPLBACKEND", "MPLCONFIGDIR", "ORIX_SHEET", "MY_SETTING", "__CF_USER_TEXT_ENCODING"]), "\(env)")334 #expect(env.contains("MY_SETTING") && !env.contains("ORIX_TEST_SECRET"))335 let home = try #require(o["home"] as? String), cwd = try #require(o["cwd"] as? String)336 #expect(home.contains(PythonPaths.runDirectoryPrefix) && cwd == home && (o["tmp"] as? String)?.contains(PythonPaths.runDirectoryPrefix) == true)337 #expect((o["path"] as? [String])?.contains { $0.hasPrefix("/tmp/evil") || $0.contains("orix-python-") } == false)338 #expect(o["flags"] as? [Int] == [1, 1, 1, 1])339 #expect(o["stdin"] as? String == "" && o["argv"] as? [String] == ["script.py"] && o["name"] as? String == "__main__" && o["restricted"] as? Bool == true)340 let fds = try #require(o["fds"] as? [Int])341 #expect(Set(fds).subtracting([0, 1, 2, 3, 4]).count <= 1, "only stdio and the two RPC descriptors are inherited (plus the one listing /dev/fd): \(fds)")342 #expect(!FileManager.default.fileExists(atPath: home))343 }344}345346@Suite(.serialized) struct AppExitTests {347 @Test(.enabled(Interp.noPython) { await Interp.any != nil })348 func quittingTheAppKillsRunningScriptsAndRemovesTheirDirectories() async throws {349 let py = try #require(await Interp.any)350 Env.prepare()351 let box = OutputBox()352 let runner = PythonRunner(interpreter: py)353 let task = Task { try await runner.run(code: "import os, subprocess, sys, time\nk = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(600)'])\nprint('PID', os.getpid()); print('PID', k.pid); print('DIR', os.getcwd(), flush=True)\ntime.sleep(600)",354 host: FakeHost(), onOutput: { box.add($0, $1) }) }355 let deadline = Date().addingTimeInterval(15)356 while !box.stdout.contains("DIR "), Date() < deadline { try await Task.sleep(nanoseconds: 20_000_000) }357 let pids = announcedPids(box.stdout)358 let cwd = try #require(box.stdout.split(separator: "\n").first { $0.hasPrefix("DIR ") }.map { String($0.dropFirst(4)) })359 let runDirectory = URL(fileURLWithPath: cwd).deletingLastPathComponent()360 #expect(pids.count == 2 && pids.allSatisfy(alive) && FileManager.default.fileExists(atPath: runDirectory.path))361 #expect(PythonRunner.runningScriptCount >= 1)362 // what the app does in applicationWillTerminate (restricted to this test's run: other suites run in parallel)363 LiveRuns.shared.killAll { PythonSandbox.canonical($0.path) == PythonSandbox.canonical(runDirectory.path) }364 #expect(await waitUntilDead(pids).isEmpty)365 #expect(!FileManager.default.fileExists(atPath: runDirectory.path))366 let r = try await task.value367 #expect(r.cancelled && r.terminationSignal == SIGKILL)368 }369370 /// The app is killed without any chance to clean up: the script notices it lost its parent and kills its own group.371 @Test(.enabled(Interp.noPython) { await Interp.any != nil }, arguments: [true, false])372 func scriptDoesNotOutliveACrashedApp(restricted: Bool) async throws {373 let py = try #require(await Interp.any)374 Env.prepare()375 // a stand-in "app": a shell that starts the runner exactly as PythonRunner does, then is SIGKILLed376 let prepared = try PreparedRun(interpreter: py, code: "import os, subprocess, sys, time\nk = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(600)'])\nopen('pids', 'w').write('%d %d' % (os.getpid(), k.pid))\ntime.sleep(600)",377 options: { var o = PythonRunOptions(); o.restricted = restricted; return o }())378 defer { prepared.removeAll() }379 func quote(_ s: String) -> String { "'" + s.replacingOccurrences(of: "'", with: "'\\''") + "'" }380 let command = ([prepared.executable] + prepared.arguments).map(quote).joined(separator: " ") + " 3</dev/null 4>/dev/null & wait"381 let app = Process()382 app.executableURL = URL(fileURLWithPath: "/bin/sh"); app.arguments = ["-c", command]383 app.environment = prepared.environment; app.currentDirectoryURL = prepared.workingDirectory384 app.standardOutput = FileHandle.nullDevice; app.standardError = FileHandle.nullDevice385 try app.run()386 let file = prepared.workingDirectory.appendingPathComponent("pids")387 let deadline = Date().addingTimeInterval(15)388 while (try? String(contentsOf: file, encoding: .utf8))?.split(separator: " ").count != 2, Date() < deadline { try await Task.sleep(nanoseconds: 50_000_000) }389 let pids = (try String(contentsOf: file, encoding: .utf8)).split(separator: " ").compactMap { Int($0) }390 try #require(pids.count == 2 && pids.allSatisfy(alive))391 kill(app.processIdentifier, SIGKILL) // the "app" dies abruptly392 #expect(await waitUntilDead(pids, seconds: 5).isEmpty, "the script and its child must exit on their own: \(pids.filter(alive))")393 }394}395