diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index d89b24ac59bec0..c7f2d0191b8cc0 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -1,5 +1,6 @@ import atexit import errno +import json import os import selectors import signal @@ -163,19 +164,19 @@ def ensure_running(self): self._forkserver_pid = None cmd = ('from multiprocessing.forkserver import main; ' + - 'main(%d, %d, %r, **%r)') + 'main(listener_fd=%d, alive_r=%d, init_r=%d)') - main_kws = {} if self._preload_modules: data = spawn.get_preparation_data('ignore') - if 'sys_path' in data: - main_kws['sys_path'] = data['sys_path'] - if 'init_main_from_path' in data: - main_kws['main_path'] = data['init_main_from_path'] - if 'sys_argv' in data: - main_kws['sys_argv'] = data['sys_argv'] - if self._preload_on_error != 'ignore': - main_kws['on_error'] = self._preload_on_error + preload_kwargs = { + "preload": self._preload_modules, + "sys_path": data["sys_path"], + "main_path": data.get("init_main_from_path", None), + "sys_argv": data["sys_argv"], + "on_error": self._preload_on_error, + } + else: + preload_kwargs = None with socket.socket(socket.AF_UNIX) as listener: address = connection.arbitrary_address('AF_UNIX') @@ -188,30 +189,31 @@ def ensure_running(self): # when they all terminate the read end becomes ready. alive_r, alive_w = os.pipe() # A short lived pipe to initialize the forkserver authkey. - authkey_r, authkey_w = os.pipe() + init_r, init_w = os.pipe() try: - fds_to_pass = [listener.fileno(), alive_r, authkey_r] - main_kws['authkey_r'] = authkey_r - cmd %= (listener.fileno(), alive_r, self._preload_modules, - main_kws) + fds_to_pass = [listener.fileno(), alive_r, init_r] + cmd %= (listener.fileno(), alive_r, init_r) exe = spawn.get_executable() args = [exe] + util._args_from_interpreter_flags() args += ['-c', cmd] pid = util.spawnv_passfds(exe, args, fds_to_pass) except: os.close(alive_w) - os.close(authkey_w) + os.close(init_w) raise finally: os.close(alive_r) - os.close(authkey_r) + os.close(init_r) # Authenticate our control socket to prevent access from # processes we have not shared this key with. try: self._forkserver_authkey = os.urandom(_AUTHKEY_LEN) - os.write(authkey_w, self._forkserver_authkey) + os.write(init_w, self._forkserver_authkey) + preload_data = json.dumps(preload_kwargs).encode() + os.write(init_w, struct.pack("Q", len(preload_data))) + os.write(init_w, preload_data) finally: - os.close(authkey_w) + os.close(init_w) self._forkserver_address = address self._forkserver_alive_fd = alive_w self._forkserver_pid = pid @@ -281,19 +283,22 @@ def _handle_preload(preload, main_path=None, sys_path=None, sys_argv=None, util._flush_std_streams() -def main(listener_fd, alive_r, preload, main_path=None, sys_path=None, - *, sys_argv=None, authkey_r=None, on_error='ignore'): +def main(listener_fd, alive_r, init_r): """Run forkserver.""" - if authkey_r is not None: - try: - authkey = os.read(authkey_r, _AUTHKEY_LEN) - assert len(authkey) == _AUTHKEY_LEN, f'{len(authkey)} < {_AUTHKEY_LEN}' - finally: - os.close(authkey_r) - else: - authkey = b'' - - _handle_preload(preload, main_path, sys_path, sys_argv, on_error) + try: + authkey = os.read(init_r, _AUTHKEY_LEN) + assert len(authkey) == _AUTHKEY_LEN, f'{len(authkey)} < {_AUTHKEY_LEN}' + + preload_data_len, = struct.unpack("Q", os.read(init_r, struct.calcsize("Q"))) + preload_data = b"" + while len(preload_data) < preload_data_len: + preload_data += os.read(init_r, preload_data_len - len(preload_data)) + preload_kwargs = json.loads(preload_data.decode()) + finally: + os.close(init_r) + + if preload_kwargs: + _handle_preload(**preload_kwargs) util._close_stdin() diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index cc07062eee6f98..34bed9ac1e26a6 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -81,6 +81,11 @@ except ImportError: msvcrt = None +try: + import resource +except ImportError: + resource = None + if support.HAVE_ASAN_FORK_BUG: # gh-89363: Skip multiprocessing tests if Python is built with ASAN to @@ -7106,6 +7111,28 @@ def test_preload_main_sys_argv(self): '', ]) + def test_preload_main_sys_argv_limits(self): + # gh-144503: Check that sys.argv is set before __main__ is pre-loaded + if multiprocessing.get_start_method() != "forkserver": + self.skipTest("forkserver specific test") + + max_str_arglen = 32 * resource.getpagesize() + argv = ["a" * (max_str_arglen - 1), "b"] + name = os.path.join(os.path.dirname(__file__), 'mp_preload_sysargv.py') + _, out, err = test.support.script_helper.assert_python_ok( + name, *argv) + self.assertEqual(err, b'') + + out = out.decode().split("\n") + expected_argv = str(argv) + self.assertEqual(out, [ + f"module:{expected_argv}", + f"fun:{expected_argv}", + f"module:{expected_argv}", + f"fun:{expected_argv}", + '', + ]) + # # Mixins # diff --git a/Misc/NEWS.d/next/Library/2026-02-05-16-29-45.gh-issue-144503.f9sl_I.rst b/Misc/NEWS.d/next/Library/2026-02-05-16-29-45.gh-issue-144503.f9sl_I.rst new file mode 100644 index 00000000000000..684ee1ab716c8b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-02-05-16-29-45.gh-issue-144503.f9sl_I.rst @@ -0,0 +1,3 @@ +Fix :mod:`multiprocessing` ``forkserver`` bug which prevented starting of +the forkserver if the total length of command line arguments in ``sys.argv`` +exceeded the maximum length of a single command line argument.