bpo-37069: regrtest uses sys.unraisablehook (GH-13759)

regrtest now uses sys.unraisablehook() to mark a test as "environment
altered" (ENV_CHANGED) if it emits an "unraisable exception".
Moreover, regrtest logs a warning in this case.

Use "python3 -m test --fail-env-changed" to catch unraisable
exceptions in tests.

Author: vstinner

Lib/test/libregrtest/setup.py

@@ -10,6 +10,8 @@
 except ImportError:
     gc = None
 
+from test.libregrtest.utils import setup_unraisable_hook
+
 
 def setup_tests(ns):
     try:
@@ -93,6 +95,8 @@ def _test_audit_hook(name, args):
             pass
         sys.addaudithook(_test_audit_hook)
 
+    setup_unraisable_hook()
+
 
 def suppress_msvcrt_asserts(verbose):
     try:

Lib/test/libregrtest/utils.py

@@ -2,6 +2,7 @@
 import os.path
 import sys
 import textwrap
+from test import support
 
 
 def format_duration(seconds):
@@ -59,3 +60,19 @@ def printlist(x, width=70, indent=4, file=None):
 
 def print_warning(msg):
     print(f"Warning -- {msg}", file=sys.stderr, flush=True)
+
+
+orig_unraisablehook = None
+
+
+def regrtest_unraisable_hook(unraisable):
+    global orig_unraisablehook
+    support.environment_altered = True
+    print_warning("Unraisable exception")
+    orig_unraisablehook(unraisable)
+
+
+def setup_unraisable_hook():
+    global orig_unraisablehook
+    orig_unraisablehook = sys.unraisablehook
+    sys.unraisablehook = regrtest_unraisable_hook

Lib/test/test_regrtest.py

@@ -499,7 +499,7 @@ def run_command(self, args, input=None, exitcode=0, **kw):
         if not input:
             input = ''
         if 'stderr' not in kw:
-            kw['stderr'] = subprocess.PIPE
+            kw['stderr'] = subprocess.STDOUT
         proc = subprocess.run(args,
                               universal_newlines=True,
                               input=input,
@@ -1124,6 +1124,34 @@ def test_garbage(self):
                                   env_changed=[testname],
                                   fail_env_changed=True)
 
+    def test_unraisable_exc(self):
+        # --fail-env-changed must catch unraisable exception
+        code = textwrap.dedent(r"""
+            import unittest
+            import weakref
+
+            class MyObject:
+                pass
+
+            def weakref_callback(obj):
+                raise Exception("weakref callback bug")
+
+            class Tests(unittest.TestCase):
+                def test_unraisable_exc(self):
+                    obj = MyObject()
+                    ref = weakref.ref(obj, weakref_callback)
+                    # call weakref_callback() which logs
+                    # an unraisable exception
+                    obj = None
+        """)
+        testname = self.create_test(code=code)
+
+        output = self.run_tests("--fail-env-changed", "-v", testname, exitcode=3)
+        self.check_executed_tests(output, [testname],
+                                  env_changed=[testname],
+                                  fail_env_changed=True)
+        self.assertIn("Warning -- Unraisable exception", output)
+
 
 class TestUtils(unittest.TestCase):
     def test_format_duration(self):

Misc/NEWS.d/next/Tests/2019-06-13-00-46-25.bpo-37069.wdktFo.rst

@@ -0,0 +1,7 @@
+regrtest now uses :func:`sys.unraisablehook` to mark a test as "environment
+altered" (ENV_CHANGED) if it emits an "unraisable exception". Moreover,
+regrtest logs a warning in this case.
+
+Use ``python3 -m test --fail-env-changed`` to catch unraisable exceptions in
+tests.
+