bpo-31961: Fix support of path-like executables in subprocess. (GH-5914)

Author: serhiy-storchaka

Doc/library/subprocess.rst

@@ -347,7 +347,8 @@ functions.
    the class uses the Windows ``CreateProcess()`` function.  The arguments to
    :class:`Popen` are as follows.
 
-   *args* should be a sequence of program arguments or else a single string.
+   *args* should be a sequence of program arguments or else a single string
+   or :term:`path-like object`.
    By default, the program to execute is the first item in *args* if *args* is
    a sequence.  If *args* is a string, the interpretation is
    platform-dependent and described below.  See the *shell* and *executable*
@@ -381,6 +382,15 @@ functions.
    manner described in :ref:`converting-argument-sequence`.  This is because
    the underlying ``CreateProcess()`` operates on strings.
 
+   .. versionchanged:: 3.6
+      *args* parameter accepts a :term:`path-like object` if *shell* is
+      ``False`` and a sequence containing path-like objects on POSIX.
+
+   .. versionchanged:: 3.8
+      *args* parameter accepts a :term:`path-like object` if *shell* is
+      ``False`` and a sequence containing bytes and path-like objects
+      on Windows.
+
    The *shell* argument (which defaults to ``False``) specifies whether to use
    the shell as the program to execute.  If *shell* is ``True``, it is
    recommended to pass *args* as a string rather than as a sequence.
@@ -436,6 +446,13 @@ functions.
    :program:`ps`.  If ``shell=True``, on POSIX the *executable* argument
    specifies a replacement shell for the default :file:`/bin/sh`.
 
+   .. versionchanged:: 3.6
+      *executable* parameter accepts a :term:`path-like object` on POSIX.
+
+   .. versionchanged:: 3.8
+      *executable* parameter accepts a bytes and :term:`path-like object`
+      on Windows.
+
    *stdin*, *stdout* and *stderr* specify the executed program's standard input,
    standard output and standard error file handles, respectively.  Valid values
    are :data:`PIPE`, :data:`DEVNULL`, an existing file descriptor (a positive
@@ -492,13 +509,19 @@ functions.
       The *pass_fds* parameter was added.
 
    If *cwd* is not ``None``, the function changes the working directory to
-   *cwd* before executing the child.  *cwd* can be a :class:`str` and
+   *cwd* before executing the child.  *cwd* can be a string, bytes or
    :term:`path-like <path-like object>` object.  In particular, the function
    looks for *executable* (or for the first item in *args*) relative to *cwd*
    if the executable path is a relative path.
 
    .. versionchanged:: 3.6
-      *cwd* parameter accepts a :term:`path-like object`.
+      *cwd* parameter accepts a :term:`path-like object` on POSIX.
+
+   .. versionchanged:: 3.7
+      *cwd* parameter accepts a :term:`path-like object` on Windows.
+
+   .. versionchanged:: 3.8
+      *cwd* parameter accepts a bytes object on Windows.
 
    If *restore_signals* is true (the default) all signals that Python has set to
    SIG_IGN are restored to SIG_DFL in the child process before the exec.

Lib/subprocess.py

@@ -521,7 +521,7 @@ def list2cmdline(seq):
     # "Parsing C++ Command-Line Arguments"
     result = []
     needquote = False
-    for arg in seq:
+    for arg in map(os.fsdecode, seq):
         bs_buf = []
 
         # Add a space to separate this argument from the others
@@ -1203,9 +1203,23 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,
 
             assert not pass_fds, "pass_fds not supported on Windows."
 
-            if not isinstance(args, str):
+            if isinstance(args, str):
+                pass
+            elif isinstance(args, bytes):
+                if shell:
+                    raise TypeError('bytes args is not allowed on Windows')
+                args = list2cmdline([args])
+            elif isinstance(args, os.PathLike):
+                if shell:
+                    raise TypeError('path-like args is not allowed when '
+                                    'shell is true')
+                args = list2cmdline([args])
+            else:
                 args = list2cmdline(args)
 
+            if executable is not None:
+                executable = os.fsdecode(executable)
+
             # Process startup details
             if startupinfo is None:
                 startupinfo = STARTUPINFO()
@@ -1262,7 +1276,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,
                                          int(not close_fds),
                                          creationflags,
                                          env,
-                                         os.fspath(cwd) if cwd is not None else None,
+                                         os.fsdecode(cwd) if cwd is not None else None,
                                          startupinfo)
             finally:
                 # Child is launched. Close the parent's copy of those pipe
@@ -1510,6 +1524,11 @@ def _execute_child(self, args, executable, preexec_fn, close_fds,
 
             if isinstance(args, (str, bytes)):
                 args = [args]
+            elif isinstance(args, os.PathLike):
+                if shell:
+                    raise TypeError('path-like args is not allowed when '
+                                    'shell is true')
+                args = [args]
             else:
                 args = list(args)
 

Lib/test/test_subprocess.py

@@ -304,6 +304,18 @@ def test_executable(self):
                                     "doesnotexist")
         self._assert_python([doesnotexist, "-c"], executable=sys.executable)
 
+    def test_bytes_executable(self):
+        doesnotexist = os.path.join(os.path.dirname(sys.executable),
+                                    "doesnotexist")
+        self._assert_python([doesnotexist, "-c"],
+                            executable=os.fsencode(sys.executable))
+
+    def test_pathlike_executable(self):
+        doesnotexist = os.path.join(os.path.dirname(sys.executable),
+                                    "doesnotexist")
+        self._assert_python([doesnotexist, "-c"],
+                            executable=FakePath(sys.executable))
+
     def test_executable_takes_precedence(self):
         # Check that the executable argument takes precedence over args[0].
         #
@@ -320,6 +332,16 @@ def test_executable_replaces_shell(self):
         # when shell=True.
         self._assert_python([], executable=sys.executable, shell=True)
 
+    @unittest.skipIf(mswindows, "executable argument replaces shell")
+    def test_bytes_executable_replaces_shell(self):
+        self._assert_python([], executable=os.fsencode(sys.executable),
+                            shell=True)
+
+    @unittest.skipIf(mswindows, "executable argument replaces shell")
+    def test_pathlike_executable_replaces_shell(self):
+        self._assert_python([], executable=FakePath(sys.executable),
+                            shell=True)
+
     # For use in the test_cwd* tests below.
     def _normalize_cwd(self, cwd):
         # Normalize an expected cwd (for Tru64 support).
@@ -358,6 +380,11 @@ def test_cwd(self):
         temp_dir = self._normalize_cwd(temp_dir)
         self._assert_cwd(temp_dir, sys.executable, cwd=temp_dir)
 
+    def test_cwd_with_bytes(self):
+        temp_dir = tempfile.gettempdir()
+        temp_dir = self._normalize_cwd(temp_dir)
+        self._assert_cwd(temp_dir, sys.executable, cwd=os.fsencode(temp_dir))
+
     def test_cwd_with_pathlike(self):
         temp_dir = tempfile.gettempdir()
         temp_dir = self._normalize_cwd(temp_dir)
@@ -1473,6 +1500,34 @@ def test_run_kwargs(self):
                              env=newenv)
         self.assertEqual(cp.returncode, 33)
 
+    def test_run_with_pathlike_path(self):
+        # bpo-31961: test run(pathlike_object)
+        # the name of a command that can be run without
+        # any argumenets that exit fast
+        prog = 'tree.com' if mswindows else 'ls'
+        path = shutil.which(prog)
+        if path is None:
+            self.skipTest(f'{prog} required for this test')
+        path = FakePath(path)
+        res = subprocess.run(path, stdout=subprocess.DEVNULL)
+        self.assertEqual(res.returncode, 0)
+        with self.assertRaises(TypeError):
+            subprocess.run(path, stdout=subprocess.DEVNULL, shell=True)
+
+    def test_run_with_bytes_path_and_arguments(self):
+        # bpo-31961: test run([bytes_object, b'additional arguments'])
+        path = os.fsencode(sys.executable)
+        args = [path, '-c', b'import sys; sys.exit(57)']
+        res = subprocess.run(args)
+        self.assertEqual(res.returncode, 57)
+
+    def test_run_with_pathlike_path_and_arguments(self):
+        # bpo-31961: test run([pathlike_object, 'additional arguments'])
+        path = FakePath(sys.executable)
+        args = [path, '-c', 'import sys; sys.exit(57)']
+        res = subprocess.run(args)
+        self.assertEqual(res.returncode, 57)
+
     def test_capture_output(self):
         cp = self.run_python(("import sys;"
                               "sys.stdout.write('BDFL'); "

Misc/NEWS.d/next/Library/2018-03-27-13-28-16.bpo-31961.GjLoYu.rst

@@ -0,0 +1,6 @@
+Added support for bytes and path-like objects in :func:`subprocess.Popen`
+on Windows.  The *args* parameter now accepts a :term:`path-like object` if
+*shell* is ``False`` and a sequence containing bytes and path-like objects.
+The *executable* parameter now accepts a bytes and :term:`path-like object`.
+The *cwd* parameter now accepts a bytes object.
+Based on patch by Anders Lorentsen.