From c6a3bea2686423250bdf772a3c86b15a591be523 Mon Sep 17 00:00:00 2001 From: Sjoerd Job Postmus Date: Thu, 5 Feb 2026 16:09:02 +0100 Subject: [PATCH 1/4] gh-144503: Pass `sys.argv` as separate command line arguments. The maximum length of a single command line argument is more restricted than the total size of all command line arguments together. --- Lib/multiprocessing/forkserver.py | 11 ++++++++--- Lib/test/_test_multiprocessing.py | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index d89b24ac59bec0..bce7b37a17886f 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -172,8 +172,6 @@ def ensure_running(self): main_kws['sys_path'] = data['sys_path'] if 'init_main_from_path' in data: main_kws['main_path'] = data['init_main_from_path'] - if 'sys_argv' in data: - main_kws['sys_argv'] = data['sys_argv'] if self._preload_on_error != 'ignore': main_kws['on_error'] = self._preload_on_error @@ -197,6 +195,8 @@ def ensure_running(self): exe = spawn.get_executable() args = [exe] + util._args_from_interpreter_flags() args += ['-c', cmd] + if self._preload_modules: + args += data["sys_argv"] pid = util.spawnv_passfds(exe, args, fds_to_pass) except: os.close(alive_w) @@ -282,7 +282,7 @@ def _handle_preload(preload, main_path=None, sys_path=None, sys_argv=None, def main(listener_fd, alive_r, preload, main_path=None, sys_path=None, - *, sys_argv=None, authkey_r=None, on_error='ignore'): + *, authkey_r=None, on_error='ignore'): """Run forkserver.""" if authkey_r is not None: try: @@ -293,6 +293,11 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None, else: authkey = b'' + if preload: + sys_argv = sys.argv[1:] + else: + sys_argv = None + _handle_preload(preload, main_path, sys_path, sys_argv, on_error) util._close_stdin() diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index cc07062eee6f98..20d5948a46479b 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -20,6 +20,7 @@ import collections.abc import socket import random +import resource import logging import shutil import subprocess @@ -7106,6 +7107,28 @@ def test_preload_main_sys_argv(self): '', ]) + def test_preload_main_sys_argv_limits(self): + # gh-144503: Check that sys.argv is set before __main__ is pre-loaded + if multiprocessing.get_start_method() != "forkserver": + self.skipTest("forkserver specific test") + + max_str_arglen = 32 * resource.getpagesize() + argv = ["a" * (max_str_arglen - 1), "b"] + name = os.path.join(os.path.dirname(__file__), 'mp_preload_sysargv.py') + _, out, err = test.support.script_helper.assert_python_ok( + name, *argv) + self.assertEqual(err, b'') + + out = out.decode().split("\n") + expected_argv = str(argv) + self.assertEqual(out, [ + f"module:{expected_argv}", + f"fun:{expected_argv}", + f"module:{expected_argv}", + f"fun:{expected_argv}", + '', + ]) + # # Mixins # From 5b8ff7bf337c4cee39a9bdcb07ca1a54db07a100 Mon Sep 17 00:00:00 2001 From: Sjoerd Job Postmus Date: Thu, 5 Feb 2026 16:29:58 +0100 Subject: [PATCH 2/4] gh-144503: Add news entry --- .../Library/2026-02-05-16-29-45.gh-issue-144503.f9sl_I.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-02-05-16-29-45.gh-issue-144503.f9sl_I.rst diff --git a/Misc/NEWS.d/next/Library/2026-02-05-16-29-45.gh-issue-144503.f9sl_I.rst b/Misc/NEWS.d/next/Library/2026-02-05-16-29-45.gh-issue-144503.f9sl_I.rst new file mode 100644 index 00000000000000..684ee1ab716c8b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-02-05-16-29-45.gh-issue-144503.f9sl_I.rst @@ -0,0 +1,3 @@ +Fix :mod:`multiprocessing` ``forkserver`` bug which prevented starting of +the forkserver if the total length of command line arguments in ``sys.argv`` +exceeded the maximum length of a single command line argument. From 0507aa09326c0d541c2e10390d88215e799f29bf Mon Sep 17 00:00:00 2001 From: Sjoerd Job Postmus Date: Thu, 5 Feb 2026 16:33:34 +0100 Subject: [PATCH 3/4] gh-144503: Accept non-importable resource The `resource` module is not available in a WASI environment. --- Lib/test/_test_multiprocessing.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 20d5948a46479b..34bed9ac1e26a6 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -20,7 +20,6 @@ import collections.abc import socket import random -import resource import logging import shutil import subprocess @@ -82,6 +81,11 @@ except ImportError: msvcrt = None +try: + import resource +except ImportError: + resource = None + if support.HAVE_ASAN_FORK_BUG: # gh-89363: Skip multiprocessing tests if Python is built with ASAN to From be2e1f050d65dec1185aa25113424cf9b3334ca8 Mon Sep 17 00:00:00 2001 From: Sjoerd Job Postmus Date: Mon, 9 Feb 2026 09:35:50 +0100 Subject: [PATCH 4/4] gh-144503: Send initialization data for preload over pipe instead of argv The allowed size for argv is limited, while the amount of data that can be sent over a pipe is virtually unlimited. --- Lib/multiprocessing/forkserver.py | 72 +++++++++++++++---------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index bce7b37a17886f..c7f2d0191b8cc0 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -1,5 +1,6 @@ import atexit import errno +import json import os import selectors import signal @@ -163,17 +164,19 @@ def ensure_running(self): self._forkserver_pid = None cmd = ('from multiprocessing.forkserver import main; ' + - 'main(%d, %d, %r, **%r)') + 'main(listener_fd=%d, alive_r=%d, init_r=%d)') - main_kws = {} if self._preload_modules: data = spawn.get_preparation_data('ignore') - if 'sys_path' in data: - main_kws['sys_path'] = data['sys_path'] - if 'init_main_from_path' in data: - main_kws['main_path'] = data['init_main_from_path'] - if self._preload_on_error != 'ignore': - main_kws['on_error'] = self._preload_on_error + preload_kwargs = { + "preload": self._preload_modules, + "sys_path": data["sys_path"], + "main_path": data.get("init_main_from_path", None), + "sys_argv": data["sys_argv"], + "on_error": self._preload_on_error, + } + else: + preload_kwargs = None with socket.socket(socket.AF_UNIX) as listener: address = connection.arbitrary_address('AF_UNIX') @@ -186,32 +189,31 @@ def ensure_running(self): # when they all terminate the read end becomes ready. alive_r, alive_w = os.pipe() # A short lived pipe to initialize the forkserver authkey. - authkey_r, authkey_w = os.pipe() + init_r, init_w = os.pipe() try: - fds_to_pass = [listener.fileno(), alive_r, authkey_r] - main_kws['authkey_r'] = authkey_r - cmd %= (listener.fileno(), alive_r, self._preload_modules, - main_kws) + fds_to_pass = [listener.fileno(), alive_r, init_r] + cmd %= (listener.fileno(), alive_r, init_r) exe = spawn.get_executable() args = [exe] + util._args_from_interpreter_flags() args += ['-c', cmd] - if self._preload_modules: - args += data["sys_argv"] pid = util.spawnv_passfds(exe, args, fds_to_pass) except: os.close(alive_w) - os.close(authkey_w) + os.close(init_w) raise finally: os.close(alive_r) - os.close(authkey_r) + os.close(init_r) # Authenticate our control socket to prevent access from # processes we have not shared this key with. try: self._forkserver_authkey = os.urandom(_AUTHKEY_LEN) - os.write(authkey_w, self._forkserver_authkey) + os.write(init_w, self._forkserver_authkey) + preload_data = json.dumps(preload_kwargs).encode() + os.write(init_w, struct.pack("Q", len(preload_data))) + os.write(init_w, preload_data) finally: - os.close(authkey_w) + os.close(init_w) self._forkserver_address = address self._forkserver_alive_fd = alive_w self._forkserver_pid = pid @@ -281,24 +283,22 @@ def _handle_preload(preload, main_path=None, sys_path=None, sys_argv=None, util._flush_std_streams() -def main(listener_fd, alive_r, preload, main_path=None, sys_path=None, - *, authkey_r=None, on_error='ignore'): +def main(listener_fd, alive_r, init_r): """Run forkserver.""" - if authkey_r is not None: - try: - authkey = os.read(authkey_r, _AUTHKEY_LEN) - assert len(authkey) == _AUTHKEY_LEN, f'{len(authkey)} < {_AUTHKEY_LEN}' - finally: - os.close(authkey_r) - else: - authkey = b'' - - if preload: - sys_argv = sys.argv[1:] - else: - sys_argv = None - - _handle_preload(preload, main_path, sys_path, sys_argv, on_error) + try: + authkey = os.read(init_r, _AUTHKEY_LEN) + assert len(authkey) == _AUTHKEY_LEN, f'{len(authkey)} < {_AUTHKEY_LEN}' + + preload_data_len, = struct.unpack("Q", os.read(init_r, struct.calcsize("Q"))) + preload_data = b"" + while len(preload_data) < preload_data_len: + preload_data += os.read(init_r, preload_data_len - len(preload_data)) + preload_kwargs = json.loads(preload_data.decode()) + finally: + os.close(init_r) + + if preload_kwargs: + _handle_preload(**preload_kwargs) util._close_stdin()