bpo-31804: Fix multiprocessing.Process with broken standard streams (#6079)

In some conditions the standard streams will be None or closed in the child process (for example if using "pythonw" instead of "python" on Windows).  Avoid failing with a non-0 exit code in those conditions.

Report and initial patch by poxthegreat.

Author: pitrou

Lib/multiprocessing/popen_fork.py

@@ -14,14 +14,7 @@ class Popen(object):
     method = 'fork'
 
     def __init__(self, process_obj):
-        try:
-            sys.stdout.flush()
-        except (AttributeError, ValueError):
-            pass
-        try:
-            sys.stderr.flush()
-        except (AttributeError, ValueError):
-            pass
+        util._flush_std_streams()
         self.returncode = None
         self.finalizer = None
         self._launch(process_obj)

Lib/multiprocessing/process.py

@@ -314,8 +314,7 @@ def _bootstrap(self):
         finally:
             threading._shutdown()
             util.info('process exiting with exitcode %d' % exitcode)
-            sys.stdout.flush()
-            sys.stderr.flush()
+            util._flush_std_streams()
 
         return exitcode
 

Lib/multiprocessing/util.py

@@ -391,6 +391,20 @@ def _close_stdin():
     except (OSError, ValueError):
         pass
 
+#
+# Flush standard streams, if any
+#
+
+def _flush_std_streams():
+    try:
+        sys.stdout.flush()
+    except (AttributeError, ValueError):
+        pass
+    try:
+        sys.stderr.flush()
+    except (AttributeError, ValueError):
+        pass
+
 #
 # Start a program with only specified fds kept open
 #

Lib/test/_test_multiprocessing.py

@@ -584,10 +584,19 @@ def test_wait_for_threads(self):
         self.assertTrue(evt.is_set())
 
     @classmethod
-    def _test_error_on_stdio_flush(self, evt):
+    def _test_error_on_stdio_flush(self, evt, break_std_streams={}):
+        for stream_name, action in break_std_streams.items():
+            if action == 'close':
+                stream = io.StringIO()
+                stream.close()
+            else:
+                assert action == 'remove'
+                stream = None
+            setattr(sys, stream_name, None)
         evt.set()
 
-    def test_error_on_stdio_flush(self):
+    def test_error_on_stdio_flush_1(self):
+        # Check that Process works with broken standard streams
         streams = [io.StringIO(), None]
         streams[0].close()
         for stream_name in ('stdout', 'stderr'):
@@ -601,6 +610,24 @@ def test_error_on_stdio_flush(self):
                     proc.start()
                     proc.join()
                     self.assertTrue(evt.is_set())
+                    self.assertEqual(proc.exitcode, 0)
+                finally:
+                    setattr(sys, stream_name, old_stream)
+
+    def test_error_on_stdio_flush_2(self):
+        # Same as test_error_on_stdio_flush_1(), but standard streams are
+        # broken by the child process
+        for stream_name in ('stdout', 'stderr'):
+            for action in ('close', 'remove'):
+                old_stream = getattr(sys, stream_name)
+                try:
+                    evt = self.Event()
+                    proc = self.Process(target=self._test_error_on_stdio_flush,
+                                        args=(evt, {stream_name: action}))
+                    proc.start()
+                    proc.join()
+                    self.assertTrue(evt.is_set())
+                    self.assertEqual(proc.exitcode, 0)
                 finally:
                     setattr(sys, stream_name, old_stream)
 

Misc/NEWS.d/next/Library/2018-03-11-19-03-52.bpo-31804.i8KUMp.rst

@@ -0,0 +1,2 @@
+Avoid failing in multiprocessing.Process if the standard streams are closed
+or None at exit.