diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 082269f..46ad88d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,12 +12,12 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] - python-version: [3.7, 3.8, 3.9, "3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Setup Python ${{ matrix.python-version }} / ${{ matrix.os }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} allow-prereleases: true diff --git a/ptyprocess/_posix_spawn.py b/ptyprocess/_posix_spawn.py new file mode 100644 index 0000000..776216a --- /dev/null +++ b/ptyprocess/_posix_spawn.py @@ -0,0 +1,320 @@ +"""Start a process in a new pseudo-terminal without forking this process. + +fork() makes a child process with only one thread: the thread that called +fork(). The child also has a copy of all the memory of this process, including +the thread-local objects of the other threads. Before the child calls `exec`, +Python frees these objects, so their finalizers run in the child. Python also +calls the functions registered with os.register_at_fork(). These finalizers +and functions come from extensions and libraries. A finalizer or a function +can wait forever for a lock, because the thread that held the lock does not +exist in the child. + +Instead of fork(), spawn() uses os.posix_spawn() to start a small helper +interpreter. The helper sets up the terminal and then calls `exec`, so no code +from this process runs in the child. The code of the helper is in +_posix_spawn_helper.py. +""" +import contextlib +import errno +import fcntl +import functools +import logging +import os +import re +import selectors +import signal +import sys +import termios +import threading +import time + +log = logging.getLogger(__name__) + +# How many seconds spawn() waits for the helper to call `exec`. spawn() blocks +# the thread that calls it, and a chdir() into a stalled network mount can +# block the helper with no time limit. +EXEC_TIMEOUT = 30 + +# When this environment variable is 1, PtyProcess.spawn() forks, as in +# ptyprocess 0.7.0. +FORK_VARIABLE = 'PTYPROCESS_USE_FORK' + + +def available(): + """Return True if spawn() can start a process without forking this + process. On Solaris, PtyProcess.spawn() keeps its own fork path. + + Return False when the environment variable PTYPROCESS_USE_FORK is 1. + Otherwise, the first call starts the helper once to test it. This test + takes 15 to 30 ms on Linux, and at most EXEC_TIMEOUT seconds. + """ + return ( + os.environ.get(FORK_VARIABLE) != '1' + and hasattr(os, 'posix_spawn') + and hasattr(termios, 'TIOCSCTTY') + and not sys.platform.startswith(('solaris', 'sunos')) + and not getattr(sys, 'frozen', False) + and bool(sys.executable) + and _helper_starts(sys.executable) + ) + + +@functools.lru_cache(maxsize=None) +def _helper_starts(executable): + """Return True if the helper starts a command when `executable` runs the + helper. + + This function starts `executable -I -S -c ""` in a new terminal, in the + same way as spawn() starts a command. For example, a relocated interpreter + that needs PYTHONHOME does not start, because the -I option makes the + interpreter ignore PYTHONHOME. A program that is not Python, for example an + application that Nuitka compiled, does not run the helper. + """ + master, tty = os.openpty() + try: + pid = spawn(master, tty, [executable, '-I', '-S', '-c', ''], + executable=executable) + except OSError as error: + log.warning('PtyProcess.spawn() forks, because %s did not run the ' + 'helper: %s', executable, error) + return False + finally: + os.close(master) + os.close(tty) + os.waitpid(pid, 0) + return True + + +@functools.lru_cache(maxsize=None) +def _helper_source(): + """Return the code of the helper, from _posix_spawn_helper.py. + + spawn() gives this code to `python -I -S -c`, because the helper cannot + import ptyprocess: with -S, sys.path does not have site-packages. + """ + # pkgutil also reads the file from a zip file. This function imports + # pkgutil, because an import at the top of the module adds about 3 ms to + # the import of ptyprocess. + import pkgutil + data = pkgutil.get_data(__package__, '_posix_spawn_helper.py') + if data is None: + raise FileNotFoundError('The loader of %s cannot read ' + '_posix_spawn_helper.py' % __package__) + return data.decode('utf-8') + + +def _encode_environment(env): + """Return `env` in the form that the helper reads from the environment + pipe. The keys and values of `env` can be str or bytes. + + Raise the same errors as os.execve() for a key or a value that os.execve() + does not accept, so that the caller of spawn() gets these errors instead of + the helper. + """ + fields = [] + for key, value in env.items(): + key, value = os.fsencode(key), os.fsencode(value) + if b'\0' in key or b'\0' in value: + raise ValueError('embedded null byte') + if not key or b'=' in key[1:]: + raise ValueError('illegal environment variable name') + fields += [key, value] + return b''.join(field + b'\0' for field in fields) + + +def _wait_for_exec(error_read, env_write, environment): + """Write `environment` to the helper, and return what the helper writes to + the error pipe. + + This function closes `env_write` after the last byte, because the helper + reads the environment pipe until EOF. It returns None when the helper does + not call `exec` within EXEC_TIMEOUT seconds. + """ + deadline = time.monotonic() + EXEC_TIMEOUT + unsent = memoryview(environment) + report = b'' + with os.fdopen(env_write, 'wb', buffering=0) as pipe, \ + selectors.DefaultSelector() as selector: + os.set_blocking(env_write, False) + selector.register(error_read, selectors.EVENT_READ) + selector.register(pipe, selectors.EVENT_WRITE) + while True: + ready = selector.select(deadline - time.monotonic()) + if not ready: + return None + for key, _ in ready: + if key.fileobj is not pipe: + data = os.read(error_read, 4096) + if not data: + return report + report += data + continue + unsent = unsent[pipe.write(unsent) or 0:] + if not unsent: + selector.unregister(pipe) + pipe.close() + + +def _above(fd, number): + """Return `fd` if it is above `number`. Otherwise, move `fd` above + `number` and return the new number.""" + if fd > number: + return fd + copy = fcntl.fcntl(fd, fcntl.F_DUPFD_CLOEXEC, number + 1) + os.close(fd) + return copy + + +def _close(*fds): + """Close each file descriptor in `fds` that is not -1.""" + for fd in fds: + if fd != -1: + os.close(fd) + + +def _kill(pid): + """Kill the helper, or the command that the helper started, and wait for + it in a thread. + + A killed helper that waits in chdir() exits only when chdir() returns, so + the thread that calls this function does not wait. + """ + os.kill(pid, signal.SIGKILL) + threading.Thread(target=os.waitpid, args=(pid, 0), daemon=True).start() + + +def _last_line(master): + """Return the last line that the helper printed to the terminal, without + colors. + + Call this function only after the helper has exited. + """ + output = b'' + os.set_blocking(master, False) + # The read raises an OSError when no output is left: EAGAIN, or EIO on + # Linux when no process has the terminal open. + with contextlib.suppress(OSError): + while True: + chunk = os.read(master, 65536) + if not chunk: + break + output += chunk + text = re.sub(r'\x1b\[[0-9;]*m', '', output.decode(errors='replace')) + lines = [line.strip() for line in text.splitlines() if line.strip()] + return lines[-1] if lines else '' + + +def _exit_code(status): + """Return the exit code in `status` from os.waitpid(), or minus the number + of the signal that ended the process.""" + if os.WIFSIGNALED(status): + return -os.WTERMSIG(status) + return os.WEXITSTATUS(status) + + +def spawn(master, tty, argv, cwd=None, env=None, pass_fds=(), + restore_signals=False, executable=None): + """Start `argv` in a new session, with `tty` as its controlling terminal, + and return the process ID. `master` is the other end of the terminal, and + `executable` is the interpreter of the helper, sys.executable by default. + + The result is the same as with pty.fork(): the child is the leader of a new + session, `tty` is the controlling terminal of the child, and file + descriptors 0, 1 and 2 of the child are `tty`. `argv[0]` must be the path + of the command. The child also keeps the file descriptors in `pass_fds` + that do not have the close-on-exec flag. The child gets `env` as its + environment, or the environment of this process when `env` is None. + + When an OSError stops the helper before `exec`, for example because the + directory `cwd` does not exist, this function raises that error. When the + helper exits before `exec` for another reason, for example because a + signal ends it, this function raises OSError with the exit code and the + last line that the helper printed. This function reads that line from + `master`. When the helper does not call `exec` within EXEC_TIMEOUT + seconds, this function kills the helper and raises TimeoutError. An + exception while this function waits, for example KeyboardInterrupt, also + kills the helper. + """ + if executable is None: + executable = sys.executable + environment = _encode_environment(os.environ if env is None else env) + pass_fds = sorted(set(pass_fds) - {0, 1, 2}) + # In the child, the error pipe and the environment pipe get the two lowest + # numbers above 2 that are not in `pass_fds`. + error_fd, env_fd = [fd for fd in range(3, len(pass_fds) + 5) + if fd not in pass_fds][:2] + defaults = [name for name in ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ') + if hasattr(signal, name) and ( + restore_signals or + signal.getsignal(getattr(signal, name)) != signal.SIG_IGN)] + # _posix_spawn_helper.py explains the arguments, the file descriptors, the + # environment and the blocked SIGINT. + helper_argv = [ + executable, '-I', '-S', '-X', 'utf8', '-c', _helper_source(), + str(error_fd), str(env_fd), ','.join(map(str, pass_fds)), cwd or '', + ','.join(defaults), + ] + list(argv) + + tty_copy = error_read = error_write = env_read = env_write = -1 + try: + # In the child, posix_spawn() copies `tty_copy`, `error_write` and + # `env_read` to file descriptors 0, 1, 2, `error_fd` and `env_fd`. The + # source of a copy must not be the target of an earlier copy. It also + # must not be its own target, because older C libraries then keep the + # close-on-exec flag. So the sources get numbers above `env_fd`. + tty_copy = fcntl.fcntl(tty, fcntl.F_DUPFD_CLOEXEC, env_fd + 1) + error_read, error_write = os.pipe() + error_write = _above(error_write, env_fd) + env_read, env_write = os.pipe() + env_read = _above(env_read, env_fd) + pid = os.posix_spawn( + executable, + helper_argv, + os.environ, + file_actions=[ + (os.POSIX_SPAWN_DUP2, tty_copy, 0), + (os.POSIX_SPAWN_DUP2, tty_copy, 1), + (os.POSIX_SPAWN_DUP2, tty_copy, 2), + (os.POSIX_SPAWN_DUP2, error_write, error_fd), + (os.POSIX_SPAWN_DUP2, env_read, env_fd), + ], + setsigmask={signal.SIGINT}, + ) + except BaseException: + _close(error_read, env_read, env_write) + raise + finally: + _close(tty_copy, error_write) + + # This process keeps its copy of the read end of the environment pipe + # until the wait ends. The pipe then always has a reader, so a write to it + # cannot raise SIGPIPE, which would end this process when SIGPIPE has its + # default action. + try: + report = _wait_for_exec(error_read, env_write, environment) + except BaseException: + _kill(pid) + raise + finally: + os.close(error_read) + os.close(env_read) + if report is None: + _kill(pid) + raise TimeoutError(errno.ETIMEDOUT, 'The command did not start ' + 'within %d s' % EXEC_TIMEOUT) + error = report[1:] if report.startswith(b'\0') else report + if error: + os.waitpid(pid, 0) + code, _, filename = error.partition(b':') + raise OSError(int(code), os.strerror(int(code)), + os.fsdecode(filename) or None) + if not report: + # The helper exited before `exec`, and no OSError stopped it. + line = _last_line(master) + _, status = os.waitpid(pid, 0) + message = 'The helper exited with code %d before %s started' % ( + _exit_code(status), os.fsdecode(argv[0])) + if line: + message += ': ' + line + raise OSError(message) + return pid diff --git a/ptyprocess/_posix_spawn_helper.py b/ptyprocess/_posix_spawn_helper.py new file mode 100644 index 0000000..29f6af4 --- /dev/null +++ b/ptyprocess/_posix_spawn_helper.py @@ -0,0 +1,123 @@ +"""Start a command in a new session, with the terminal as its controlling +terminal. + +ptyprocess._posix_spawn starts the helper with os.posix_spawn(), as +`python -I -S -X utf8 -c CODE error_fd env_fd pass_fds cwd defaults command +[arg ...]`. CODE is the text of this file. File descriptors 0, 1 and 2 are the +terminal, `error_fd` is the write end of the error pipe, `env_fd` is the read +end of the environment pipe, `pass_fds` lists the other file descriptors that +the command keeps, and `defaults` lists the signals that the helper sets to +SIG_DFL. `cwd` is empty when the command starts in the directory of the +caller. With -X utf8, `cwd` and the arguments keep their bytes in every +locale. Without -X utf8, Python 3.14 and older can change or reject some of +these bytes in a legacy multibyte locale, for example ja_JP.eucJP. + +Right before `exec`, the helper writes a NUL byte to the error pipe. When an +OSError stops the helper, the helper writes `errno:filename` and exits. So a +failed `exec` gives the NUL byte and then the error. EOF with no data means +that the helper exited for another reason, for example because a signal ended +it. + +The helper runs with the environment of the caller, because +_posix_spawn.available() checked the interpreter with this environment. With +the environment of the command, the helper could miss a variable that the +interpreter needs (LD_LIBRARY_PATH), start differently even with -I +(PYTHONEXECUTABLE), or change LC_CTYPE before `exec` (PEP 538). The +environment of the command goes through the environment pipe, because other +users can read the arguments of a process. +""" + +# Every import adds to the time that the command takes to start, so the helper +# imports only the modules that it needs. +import fcntl +import os +import signal +import sys +import termios + + +def main(): + """Set up the terminal and start the command, or write the error to the + error pipe.""" + error_fd, env_fd, pass_fds, cwd, defaults, command, *args = sys.argv[1:] + error_fd, env_fd = int(error_fd), int(env_fd) + keep = {0, 1, 2, error_fd} + keep.update(map(int, filter(None, pass_fds.split(',')))) + try: + # The environment pipe has a key, a NUL byte, a value and a NUL byte + # for each variable. split() gives an empty field after the last NUL + # byte, and zip() drops this field. + data = b'' + while True: + chunk = os.read(env_fd, 65536) + if not chunk: + break + data += chunk + fields = data.split(b'\0') + env = dict(zip(fields[::2], fields[1::2])) + + # Make the terminal the controlling terminal of a new session, as + # pty.fork() does. + os.setsid() + fcntl.ioctl(0, termios.TIOCSCTTY, 0) + if cwd: + os.chdir(cwd) + + # Until setsid(), the helper is in the process group of the caller, + # and Ctrl-C or a kernel interrupt from jupyter_client sends SIGINT to + # the whole group. So _posix_spawn starts the helper with SIGINT + # blocked. SIG_IGN discards a pending SIGINT. The helper then restores + # the previous action, so the command gets the SIGINT action of the + # caller, as after fork(). os.posix_spawn(setsid=True) would avoid the + # pending SIGINT, but the builds of Python that uv installs do not + # support it. + previous = signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGINT, previous) + # Python ignores SIGPIPE, SIGXFZ and SIGXFSZ when it starts, and `exec` + # keeps ignored signals ignored. So `defaults` names each of these + # signals that the caller does not ignore, or all of them with + # `restore_signals`. The command then gets the same actions as after + # fork(). + for name in filter(None, defaults.split(',')): + signal.signal(getattr(signal, name), signal.SIG_DFL) + signal.pthread_sigmask(signal.SIG_SETMASK, ()) + + # Close the file descriptors that the command does not keep, including + # the environment pipe. The helper finds the open file descriptors in + # /proc/self/fd, or in /dev/fd on macOS, because os.closerange() can + # take more than 100 ms: it calls close() for every number up to the + # limit when close_range() is not available. On the BSDs, /dev/fd can + # be a static directory that lists only 0, 1 and 2, so the helper does + # not read /dev/fd there and uses os.closerange(). + try: + fds = os.listdir('/dev/fd' if sys.platform == 'darwin' + else '/proc/self/fd') + except OSError: + max_fd = os.sysconf('SC_OPEN_MAX') + bounds = sorted(keep) + [max_fd if 0 < max_fd < 1048576 + else 1048576] + for low, high in zip(bounds, bounds[1:]): + os.closerange(low + 1, high) + else: + for fd in map(int, fds): + if fd not in keep: + # The list includes the file descriptor of the directory, + # and os.listdir() has already closed it. + try: + os.close(fd) + except OSError: + pass + + # Because of FD_CLOEXEC, `exec` closes the error pipe. The caller then + # reads the NUL byte and EOF. + fcntl.fcntl(error_fd, fcntl.F_SETFD, fcntl.FD_CLOEXEC) + os.write(error_fd, b'\0') + os.execve(command, [command, *args], env) + except OSError as e: + os.write(error_fd, + b'%d:' % (e.errno or 0) + os.fsencode(e.filename or '')) + os._exit(127) + + +if __name__ == '__main__': + main() diff --git a/ptyprocess/ptyprocess.py b/ptyprocess/ptyprocess.py index daf20b1..933dec5 100644 --- a/ptyprocess/ptyprocess.py +++ b/ptyprocess/ptyprocess.py @@ -20,6 +20,7 @@ from pty import (STDIN_FILENO, CHILD) from .util import which, PtyProcessError +from . import _posix_spawn _platform = sys.platform.lower() @@ -185,15 +186,34 @@ def __init__(self, pid, fd): @classmethod def spawn( cls, argv, cwd=None, env=None, echo=True, preexec_fn=None, - dimensions=(24, 80), pass_fds=(), **kwargs): + dimensions=(24, 80), pass_fds=(), restore_signals=False, + **kwargs): '''Start the given command in a child process in a pseudo terminal. This does all the fork/exec type of stuff for a pty, and returns an instance of PtyProcess. + When Python has :func:`os.posix_spawn` and preexec_fn is not supplied, + this does not fork the current process. It starts a small helper + interpreter (``sys.executable -I -S -X utf8``) with + :func:`os.posix_spawn`, and the helper sets up the terminal and then + calls ``exec``. The child then has no copy of the memory of the + current process, so no code from the current process runs in the + child. On Solaris, in frozen applications, when the helper interpreter + does not start, and when the environment variable + ``PTYPROCESS_USE_FORK`` is ``1``, this forks. + If preexec_fn is supplied, it will be called with no arguments in the child process before exec-ing the specified command. It may, for instance, set signal handlers to SIG_DFL or SIG_IGN. + A preexec_fn can only run in a fork of the current process, so this + always forks when preexec_fn is supplied. + + If restore_signals is true, the signals that Python ignores (SIGPIPE, + SIGXFZ and SIGXFSZ) are reset to their default action in the child + before exec, as with :class:`subprocess.Popen`. Otherwise the child + gets the action that the current process has for each of these + signals. Dimensions of the psuedoterminal used for the subprocess can be specified as a tuple (rows, cols), or the default (24, 80) will be used. @@ -224,6 +244,27 @@ def spawn( command = command_with_path argv[0] = command + if preexec_fn is None and _posix_spawn.available(): + fd, tty = os.openpty() + try: + _setwinsize(tty, *dimensions) + if not echo: + _setecho(tty, False) + pid = _posix_spawn.spawn(fd, tty, argv, cwd, env, pass_fds, + restore_signals) + except BaseException: + os.close(fd) + raise + finally: + os.close(tty) + inst = cls(pid, fd, **kwargs) + inst.argv = argv + if env is not None: + inst.env = env + if cwd is not None: + inst.launch_dir = cwd + return inst + # [issue #119] To prevent the case where exec fails and the user is # stuck interacting with a python child process instead of whatever # was expected, we implement the solution from @@ -259,6 +300,11 @@ def spawn( if err.args[0] not in (errno.EINVAL, errno.ENOTTY): raise + if restore_signals: + for name in ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ'): + if hasattr(signal, name): + signal.signal(getattr(signal, name), signal.SIG_DFL) + # [issue #119] 3. The child closes the reading end and sets the # close-on-exec flag for the writing end. os.close(exec_err_pipe_read) diff --git a/pyproject.toml b/pyproject.toml index bca7599..b0e8997 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ authors = [ { name = "Thomas Kluyver", email = "thomas@kluyver.me.uk" }, ] readme = "README.rst" -requires-python = ">=3.7" +requires-python = ">=3.10" dynamic = ["version", "description"] classifiers = [ "Development Status :: 5 - Production/Stable", diff --git a/tests/test_fork_hazards.py b/tests/test_fork_hazards.py new file mode 100644 index 0000000..1583a22 --- /dev/null +++ b/tests/test_fork_hazards.py @@ -0,0 +1,150 @@ +"""Starting a process must not run code that belongs to other threads. + +fork() makes a child process with only one thread: the thread that called +fork(). The child also has a copy of all the memory of this process, including +the thread-local objects of the other threads. Before the command starts, +Python frees these objects, so their finalizers run in the child. A finalizer +can then wait forever for a lock, or change files that this process still +uses. +""" +import os +import subprocess +import sys +import tempfile +import textwrap +import unittest + +from ptyprocess import _posix_spawn + +# The root of this repository, which the scripts import ptyprocess from +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def run_script(script, cwd): + """Run `script` in a new Python process and return its output.""" + env = dict(os.environ) + env['PYTHONPATH'] = os.pathsep.join( + filter(None, [ROOT, env.get('PYTHONPATH')])) + argv = [sys.executable, '-c', textwrap.dedent(script)] + result = subprocess.run(argv, cwd=cwd, env=env, capture_output=True, + text=True, timeout=120) + assert result.returncode == 0, result.stderr + return result.stdout + + +@unittest.skipUnless( + _posix_spawn.available(), + '_posix_spawn.available() is False, so spawn() still forks') +class ForkHazardsTestCase(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.cwd = directory.name + + def test_lock_held_by_another_thread_does_not_hang(self): + script = r""" + import logging + import os + import threading + import time + from concurrent.futures import ThreadPoolExecutor + + from ptyprocess import PtyProcess + + # A child that hangs does not call `exec`, so kill it after 15 s to + # end the test. + forked = [] + + + def recording(fork): + def wrapper(): + result = fork() + pid = result[0] if isinstance(result, tuple) else result + if pid: + forked.append(pid) + return result + return wrapper + + + os.fork = recording(os.fork) + os.forkpty = recording(os.forkpty) + + + def watchdog(): + time.sleep(15) + print("timed out", flush=True) + for pid in forked: + os.kill(pid, 9) + + + threading.Thread(target=watchdog, daemon=True).start() + + logging.basicConfig(filename="app.log", level=logging.INFO) + log = logging.getLogger("library") + + + class Session: + def __del__(self): + log.info("session closed") + + + local = threading.local() + + + def handle_request(): + local.session = Session() + + + def log_progress(): + while True: + log.info("working") + + + executor = ThreadPoolExecutor() + executor.submit(handle_request).result() + threading.Thread(target=log_progress, daemon=True).start() + time.sleep(1) + + PtyProcess.spawn(["true"]) + print("started", flush=True) + os._exit(0) + """ + self.assertEqual(run_script(script, self.cwd), 'started\n') + + def test_objects_of_other_threads_are_not_finalized(self): + script = r""" + import os + import tempfile + import threading + from concurrent.futures import ThreadPoolExecutor + + from ptyprocess import PtyProcess + + local = threading.local() + + + def export_row(i): + if not hasattr(local, "workdir"): + local.workdir = tempfile.TemporaryDirectory(dir=".") + local.out = open("export.csv", "w") + local.out.write(f"row {i}\n") + return local.workdir.name + + + def finish(): + local.out.close() + + + executor = ThreadPoolExecutor(max_workers=1) + for i in range(3): + workdir = executor.submit(export_row, i).result() + + PtyProcess.spawn(["true"]) + executor.submit(finish).result() + print(os.path.exists(workdir)) + with open("export.csv") as f: + print(f.read().splitlines(), flush=True) + os._exit(0) + """ + self.assertEqual(run_script(script, self.cwd), + "True\n['row 0', 'row 1', 'row 2']\n") diff --git a/tests/test_posix_spawn.py b/tests/test_posix_spawn.py new file mode 100644 index 0000000..43c518c --- /dev/null +++ b/tests/test_posix_spawn.py @@ -0,0 +1,606 @@ +"""Tests for starting a process without forking this process.""" +import errno +import os +import pty +import selectors +import shlex +import shutil +import signal +import subprocess +import sys +import tempfile +import threading +import time +import unittest +import warnings +from unittest import mock + +from ptyprocess import PtyProcess, PtyProcessUnicode, _posix_spawn + +requires_helper = unittest.skipUnless( + _posix_spawn.available(), + '_posix_spawn.available() is False, so spawn() still forks') + +# The scripts below import ptyprocess from the root of this repository. +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# This program prints "ready", then sleeps for 30 s or until a signal +# stops it. +SLEEPER = ['sh', '-c', 'echo ready; exec sleep 30'] + +# This program prints the file descriptors above 2 that it has open. +LIST_OPEN_DESCRIPTORS = """\ +import os +for fd in range(3, 1024): + try: + os.fstat(fd) + except OSError: + continue + print(fd) +""" + +# This code runs in the helper before the code of _posix_spawn_helper.py. The +# helper then sees the file descriptor directories of FreeBSD without fdescfs: +# there is no /proc/self/fd, and /dev/fd lists only 0, 1 and 2. +FREEBSD_WITHOUT_FDESCFS = """\ +import os, sys +sys.platform = "freebsd14" +real_listdir = os.listdir +def listdir(path): + if path == "/proc/self/fd": + raise FileNotFoundError(path) + return ["0", "1", "2"] if path == "/dev/fd" else real_listdir(path) +os.listdir = listdir +""" + +# This script runs in a new interpreter with file descriptors 0 to 3 closed, +# so the terminal and the error pipe of the first process get numbers that +# the child replaces. The script writes the file name in the error of the +# first process and the output of a second process to result.txt. +WITHOUT_STANDARD_DESCRIPTORS = """\ +import os +os.closerange(0, 4) +from ptyprocess import PtyProcessUnicode +output = "" +try: + PtyProcessUnicode.spawn(["sh"], cwd="missing") +except FileNotFoundError as error: + output += error.filename + " " +process = PtyProcessUnicode.spawn(["sh", "-c", "echo hello"]) +try: + while True: + output += process.read(65536) +except EOFError: + pass +with open("result.txt", "w") as f: + f.write(output) +""" + +# This script runs in a new interpreter. It opens result.txt as file +# descriptor 3, and passes file descriptor 3 to a shell that writes "hello" +# to the file. +PASS_DESCRIPTOR_3 = """\ +import os +from ptyprocess import PtyProcess +fd = os.open("result.txt", os.O_WRONLY | os.O_CREAT) +if fd != 3: + os.dup2(fd, 3) + os.close(fd) +os.set_inheritable(3, True) +process = PtyProcess.spawn(["sh", "-c", "echo hello >&3"], pass_fds=(3,)) +process.wait() +""" + +# This script runs in a new interpreter with SIGPIPE at its default action. +# The helper is the program in the first argument, and that program exits +# before it reads the environment. The script writes the error from spawn() +# to result.txt. +SIGPIPE_AT_DEFAULT = """\ +import os, signal, sys +signal.signal(signal.SIGPIPE, signal.SIG_DFL) +sys.executable = sys.argv[1] +from ptyprocess import PtyProcess, _posix_spawn +# The program fails the check in available(), so the script skips it. +_posix_spawn.available = lambda: True +env = dict(os.environ, LARGE="x" * 100000) +try: + PtyProcess.spawn(["true"], env=env) +except OSError as error: + with open("result.txt", "w") as f: + f.write(str(error)) +""" + +# This script runs in a new interpreter in its own session. The interpreter +# handles SIGINT, as a server does, and sends SIGINT to its own process group +# right after posix_spawn() returns. The script writes the output of the +# command to result.txt. +CTRL_C_IN_THE_CALLER = """\ +import os, signal +from ptyprocess import PtyProcessUnicode, _posix_spawn +signal.signal(signal.SIGINT, lambda *args: None) +# The script runs the check in available() now, so that the check does not +# get the SIGINT. +assert _posix_spawn.available() +real_posix_spawn = os.posix_spawn +def posix_spawn(*args, **kwargs): + pid = real_posix_spawn(*args, **kwargs) + os.killpg(os.getpgrp(), signal.SIGINT) + return pid +os.posix_spawn = posix_spawn +process = PtyProcessUnicode.spawn(["sh", "-c", "echo started"]) +output = "" +try: + while True: + output += process.read(65536) +except EOFError: + pass +with open("result.txt", "w") as f: + f.write(output) +""" + +# This script runs in a new interpreter that ignores SIGINT, as a job that a +# shell starts with `&` does. The script starts a shell that sends SIGINT to +# itself, first with the helper and then with the fork, and writes the output +# of both shells to result.txt. +SIGINT_IGNORED = """\ +import signal +signal.signal(signal.SIGINT, signal.SIG_IGN) +from ptyprocess import PtyProcessUnicode, _posix_spawn +def output_of(argv): + process = PtyProcessUnicode.spawn(argv) + output = "" + try: + while True: + output += process.read(65536) + except EOFError: + pass + return output +argv = ["sh", "-c", "kill -INT $$; echo survived"] +output = output_of(argv) +_posix_spawn.available = lambda: False +output += output_of(argv) +with open("result.txt", "w") as f: + f.write(output) +""" + +# This script runs in a new interpreter with SIGPIPE and SIGXFSZ at their +# default action, as some command-line programs set them. The script starts +# shells that send these signals to themselves, first with the helper and then +# with the fork, and writes the output of the shells to result.txt. +SIGNALS_AT_DEFAULT = """\ +import signal +signal.signal(signal.SIGPIPE, signal.SIG_DFL) +signal.signal(signal.SIGXFSZ, signal.SIG_DFL) +from ptyprocess import PtyProcessUnicode, _posix_spawn +def output_of(argv): + process = PtyProcessUnicode.spawn(argv) + output = "" + try: + while True: + output += process.read(65536) + except EOFError: + pass + return output +output = "" +for fork in [False, True]: + if fork: + _posix_spawn.available = lambda: False + for name in ["PIPE", "XFSZ"]: + script = "ulimit -c 0; kill -%s $$; echo survived" % name + output += output_of(["sh", "-c", script]) +with open("result.txt", "w") as f: + f.write(output) +""" + + +def read_until_eof(process): + """Return everything that the process prints until it exits.""" + chunks = [] + while True: + try: + chunks.append(process.read(65536)) + except EOFError: + return ''.join(chunks) + + +def output_of(argv, **kwargs): + """Run `argv` in a new terminal and return everything that it printed.""" + process = PtyProcessUnicode.spawn(argv, **kwargs) + output = read_until_eof(process) + process.close() + return output + + +def open_descriptors(): + """Return the file descriptors above 2 that are open in a program in a new + terminal. + + While the terminal starts, this process has an inheritable file descriptor + open. + """ + read_end, write_end = os.pipe() + os.set_inheritable(write_end, True) + try: + return output_of([sys.executable, '-c', LIST_OPEN_DESCRIPTORS]).split() + finally: + os.close(read_end) + os.close(write_end) + + +def stop_with_ctrl_c(process): + """Press Ctrl-C when the program in the terminal is ready, and return its + signal.""" + output = '' + while 'ready' not in output: + output += process.read(65536) + process.write('\x03') + read_until_eof(process) + process.wait() + status = process.signalstatus + process.close() + return status + + +def survives(signal_name, **kwargs): + """Return True if a shell in a new terminal survives a signal that it sends + to itself.""" + # The default action of SIGXFSZ writes a core file, so turn core files off. + script = 'ulimit -c 0; kill -%s $$; echo survived' % signal_name + return 'survived' in output_of(['sh', '-c', script], **kwargs) + + +def open_fds(): + """Return the numbers of the file descriptors that are open in this + process.""" + return set(os.listdir('/dev/fd')) + + +def has_exited(pid): + """Return True if the process `pid` is gone within 10 s. + + A process is gone when it has exited and its parent has waited for it. + """ + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + time.sleep(0.01) + return False + + +def run_script(script, cwd, *args, **kwargs): + """Run `script` with `args` in a new Python process that imports + ptyprocess from this repository. `kwargs` go to subprocess.run().""" + env = dict(os.environ) + env['PYTHONPATH'] = os.pathsep.join( + filter(None, [ROOT, env.get('PYTHONPATH')])) + subprocess.run([sys.executable, '-c', script, *args], cwd=cwd, env=env, + timeout=60, check=True, **kwargs) + + +class PosixSpawnTestCase(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.tmp = directory.name + + def fake_python(self, script, name='python'): + """Return the path of a shell script that runs `script`.""" + path = os.path.join(self.tmp, name) + with open(path, 'w') as f: + f.write('#!/bin/sh\n' + script) + os.chmod(path, 0o755) + return path + + @requires_helper + def test_does_not_fork(self): + def fail(): + raise AssertionError('spawn() forked') + + with mock.patch.object(os, 'fork', fail), \ + mock.patch.object(os, 'forkpty', fail), \ + mock.patch.object(pty, 'fork', fail): + self.assertIn('hello', output_of(['sh', '-c', 'echo hello'])) + + def test_cwd(self): + output = output_of(['sh', '-c', 'pwd -P'], cwd=self.tmp) + self.assertEqual(output.split(), [os.path.realpath(self.tmp)]) + + def test_env(self): + env = dict(os.environ, GREETING='hello') + output = output_of(['sh', '-c', 'echo $GREETING'], env=env) + self.assertEqual(output.split(), ['hello']) + + def test_c_locale_in_env_reaches_the_command_unchanged(self): + # When env selects the C locale, CPython sets LC_CTYPE in its own + # environment (PEP 538), but the command must get the value from env. + script = 'echo "${LC_CTYPE-unset}"' + path = os.environ['PATH'] + for env, expected in [ + ({'PATH': path}, 'unset'), + ({'PATH': path, 'LC_CTYPE': 'C'}, 'C'), + ({b'PATH': os.fsencode(path), b'LC_CTYPE': b'C'}, 'C'), + ]: + with self.subTest(env=env): + output = output_of(['sh', '-c', script], env=env) + self.assertEqual(output.split(), [expected]) + + def test_pyvenv_launcher_in_env_reaches_the_command_unchanged(self): + # Since Python 3.11, CPython removes __PYVENV_LAUNCHER__ from its own + # environment when it starts. + env = dict(os.environ, __PYVENV_LAUNCHER__=sys.executable) + script = 'echo "${__PYVENV_LAUNCHER__-unset}"' + output = output_of(['sh', '-c', script], env=env) + self.assertEqual(output.strip(), sys.executable) + + def test_large_env(self): + # The environment is larger than the buffer of a pipe. + env = dict(os.environ) + env.update(('LARGE_%d' % i, str(i) * 30000) for i in range(5)) + script = 'echo ${#LARGE_0} ${#LARGE_4}' + output = output_of(['sh', '-c', script], env=env) + self.assertEqual(output.split(), ['30000', '30000']) + + def test_session_leader_with_controlling_terminal(self): + script = ( + "import os; os.close(os.open('/dev/tty', os.O_RDWR));" + "print(os.getsid(0) == os.getpid(), os.getpgrp() == os.getpid())" + ) + output = output_of([sys.executable, '-c', script]) + self.assertEqual(output.split(), ['True', 'True']) + + def test_ctrl_c_interrupts_the_program(self): + process = PtyProcessUnicode.spawn(SLEEPER) + self.assertEqual(stop_with_ctrl_c(process), signal.SIGINT) + + @requires_helper + def test_blocked_signals_are_not_inherited(self): + # The program must not inherit the signal mask of the thread that + # starts it. + processes = [] + + def start(): + signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGINT}) + processes.append(PtyProcessUnicode.spawn(SLEEPER)) + + thread = threading.Thread(target=start) + thread.start() + thread.join() + self.assertEqual(stop_with_ctrl_c(processes[0]), signal.SIGINT) + + @requires_helper + def test_ctrl_c_in_the_caller_does_not_stop_the_helper(self): + # The script runs in its own session, so that its SIGINT does not reach + # the tests. + run_script(CTRL_C_IN_THE_CALLER, self.tmp, start_new_session=True) + with open(os.path.join(self.tmp, 'result.txt')) as f: + self.assertEqual(f.read().split(), ['started']) + + def test_ignored_sigint_stays_ignored(self): + run_script(SIGINT_IGNORED, self.tmp) + with open(os.path.join(self.tmp, 'result.txt')) as f: + self.assertEqual(f.read().split(), ['survived', 'survived']) + + def test_window_size(self): + output = output_of(['stty', 'size'], dimensions=(30, 100)) + self.assertEqual(output.split(), ['30', '100']) + + def test_bytes_argv(self): + # pexpect gives bytes when it has an encoding. + sh = os.fsencode(shutil.which('sh')) + output = output_of([sh, b'-c', b'echo hello']) + self.assertEqual(output.split(), ['hello']) + + def assert_restores_signals(self): + for name in ['PIPE', 'XFSZ']: + with self.subTest(signal=name): + self.assertTrue(survives(name)) + self.assertFalse(survives(name, restore_signals=True)) + + def test_restore_signals(self): + self.assert_restores_signals() + + def test_signal_actions_of_the_caller(self): + # A shell that ignores the signal prints "survived". + run_script(SIGNALS_AT_DEFAULT, self.tmp) + with open(os.path.join(self.tmp, 'result.txt')) as f: + self.assertEqual(f.read().split(), []) + + def test_restore_signals_after_fork(self): + with mock.patch.object(_posix_spawn, 'available', return_value=False): + self.assert_restores_signals() + + def test_file_descriptors_are_not_inherited(self): + self.assertEqual(open_descriptors(), []) + + @unittest.skipUnless( + sys.implementation.name == 'cpython' + and sys.platform.startswith(('linux', 'darwin')) + and hasattr(os, 'posix_spawn'), + 'This check runs only with CPython on Linux and macOS') + def test_helper_is_available(self): + # The tests of the helper skip when available() is False. So a helper + # that does not start fails this test instead. The check runs again + # here, so that the log of this test gives the reason. + _posix_spawn._helper_starts.cache_clear() + self.assertTrue(_posix_spawn._helper_starts(sys.executable)) + + @requires_helper + def test_pass_fds_without_a_list_of_open_descriptors(self): + # On the BSDs, the helper closes file descriptors with os.closerange(). + path = os.path.join(self.tmp, 'result.txt') + fd = os.open(path, os.O_WRONLY | os.O_CREAT) + self.addCleanup(os.close, fd) + os.set_inheritable(fd, True) + code = 'import os; os.write(%d, b"hello")' % fd + helper = FREEBSD_WITHOUT_FDESCFS + _posix_spawn._helper_source() + with mock.patch.object(_posix_spawn, '_helper_source', lambda: helper): + output_of([sys.executable, '-c', code], pass_fds=(fd,)) + with open(path) as f: + self.assertEqual(f.read(), 'hello') + + @requires_helper + def test_static_dev_fd_is_not_trusted(self): + helper = FREEBSD_WITHOUT_FDESCFS + _posix_spawn._helper_source() + with mock.patch.object(_posix_spawn, '_helper_source', lambda: helper): + self.assertEqual(open_descriptors(), []) + + @requires_helper + def test_pass_descriptor_3(self): + # In the child, the error pipe is not file descriptor 3 then. + run_script(PASS_DESCRIPTOR_3, self.tmp) + with open(os.path.join(self.tmp, 'result.txt')) as f: + self.assertEqual(f.read(), 'hello\n') + + @requires_helper + def test_without_standard_file_descriptors(self): + run_script(WITHOUT_STANDARD_DESCRIPTORS, self.tmp) + with open(os.path.join(self.tmp, 'result.txt')) as f: + self.assertEqual(f.read().split(), ['missing', 'hello']) + + def test_invalid_executable(self): + program = os.path.join(self.tmp, 'not-a-program') + with open(program, 'w') as f: + f.write('not a program') + os.chmod(program, 0o755) + with self.assertRaises(OSError) as caught: + PtyProcess.spawn([program]) + self.assertEqual(caught.exception.errno, errno.ENOEXEC) + + @requires_helper + def test_helper_that_exits_before_exec(self): + # For example, the interpreter does not start, or a signal ends the + # helper. + for code, message in [ + ("raise RuntimeError('broken')", + 'code 1 before .*: RuntimeError: broken$'), + ('import os; os.kill(os.getpid(), 9)', 'code -9 before'), + ]: + helper = code + '\n' + _posix_spawn._helper_source() + with self.subTest(code=code), \ + mock.patch.object(_posix_spawn, '_helper_source', + lambda: helper): + with self.assertRaisesRegex(OSError, message): + PtyProcess.spawn(['sh', '-c', 'echo started']) + + @requires_helper + def test_invalid_env_raises_in_the_caller(self): + # On the fork path, the child of the fork gets this error and runs the + # rest of the tests. + for env, message in [ + ({'A=B': '1'}, 'illegal environment variable name'), + ({'A': '1\0'}, 'embedded null byte'), + ]: + with self.subTest(env=env): + with self.assertRaisesRegex(ValueError, message): + PtyProcess.spawn(['sh'], env=env) + + @requires_helper + def test_env_without_a_variable_that_python_needs(self): + # This wrapper needs a variable from the environment of this process to + # start Python. In the same way, an interpreter can need + # LD_LIBRARY_PATH to find its shared libpython. `env` does not have the + # variable. + python = self.fake_python('exec "$PTYPROCESS_PYTHON" "$@"\n') + env = dict(os.environ) + with mock.patch.dict(os.environ, PTYPROCESS_PYTHON=sys.executable), \ + mock.patch.object(sys, 'executable', python): + output = output_of(['sh', '-c', 'echo hello'], env=env) + self.assertEqual(output.split(), ['hello']) + + @requires_helper + def test_helper_exit_does_not_raise_sigpipe(self): + # The helper exits before it reads the environment, which is larger + # than the buffer of a pipe. The caller has SIGPIPE at its default + # action, so a write to a pipe with no reader ends the caller. + python = self.fake_python('exit 1\n') + run_script(SIGPIPE_AT_DEFAULT, self.tmp, python) + with open(os.path.join(self.tmp, 'result.txt')) as f: + self.assertRegex(f.read(), '^The helper exited with code 1 before') + + @requires_helper + def test_missing_cwd(self): + missing = os.path.join(self.tmp, 'missing') + with self.assertRaises(FileNotFoundError) as caught: + PtyProcess.spawn(['sh'], cwd=missing) + self.assertEqual(caught.exception.filename, missing) + + @requires_helper + def test_waiting_for_the_helper_has_a_timeout(self): + # This wrapper runs Python as a child, so it keeps the error pipe open + # while the command runs. A helper that waits in chdir() for a stalled + # network mount does the same. + python = self.fake_python('%s "$@"\n' % shlex.quote(sys.executable)) + with mock.patch.object(sys, 'executable', python), \ + mock.patch.object(_posix_spawn, 'EXEC_TIMEOUT', 1): + start = time.monotonic() + with self.assertRaises(TimeoutError): + PtyProcess.spawn(['sleep', '5']) + self.assertLess(time.monotonic() - start, 4) + + @requires_helper + def test_failed_start_leaves_no_descriptor_and_no_helper(self): + # For example, this process has too many open files when it opens the + # error pipe, or when it waits for the helper. + helpers = [] + real_posix_spawn = os.posix_spawn + + def posix_spawn(*args, **kwargs): + helpers.append(real_posix_spawn(*args, **kwargs)) + return helpers[-1] + + def fail(*args): + raise OSError(errno.EMFILE, os.strerror(errno.EMFILE)) + + for module, name in [(os, 'pipe'), (selectors, 'DefaultSelector')]: + del helpers[:] + with self.subTest(call=name): + before = open_fds() + with mock.patch.object(os, 'posix_spawn', posix_spawn), \ + mock.patch.object(module, name, fail): + with self.assertRaises(OSError) as caught: + PtyProcess.spawn(SLEEPER) + self.assertEqual(caught.exception.errno, errno.EMFILE) + self.assertEqual(open_fds(), before) + self.assertTrue(all(map(has_exited, helpers))) + + def test_preexec_fn_forks(self): + with mock.patch.object(_posix_spawn, 'spawn') as spawn: + output = output_of(['sh', '-c', 'echo hello'], + preexec_fn=lambda: None) + self.assertIn('hello', output) + spawn.assert_not_called() + + @requires_helper + def test_falls_back_to_fork_when_the_helper_does_not_start(self): + # For example, a relocated interpreter that needs PYTHONHOME does not + # start with -I. A program that is not Python, for example an + # application that Nuitka compiled, can exit with 0 or not exit at all. + for i, script in enumerate(['exit 1', 'exit 0', 'exec sleep 30']): + # available() keeps its result for each path, so each script needs + # its own path. + python = self.fake_python(script + '\n', 'python%d' % i) + with self.subTest(script=script), \ + mock.patch.object(sys, 'executable', python), \ + mock.patch.object(_posix_spawn, 'EXEC_TIMEOUT', 1), \ + self.assertLogs('ptyprocess._posix_spawn', 'WARNING'), \ + warnings.catch_warnings(): + warnings.filterwarnings( + 'ignore', 'This process .* is multi-threaded', + DeprecationWarning) + output = output_of(['sh', '-c', 'echo hello']) + self.assertIn('hello', output) + + @requires_helper + def test_environment_variable_selects_the_fork(self): + with mock.patch.dict(os.environ, PTYPROCESS_USE_FORK='1'), \ + mock.patch.object(_posix_spawn, 'spawn') as spawn, \ + warnings.catch_warnings(): + warnings.filterwarnings( + 'ignore', 'This process .* is multi-threaded', + DeprecationWarning) + output = output_of(['sh', '-c', 'echo hello']) + self.assertIn('hello', output) + spawn.assert_not_called()