bpo-36829: Add sys.unraisablehook() (GH-13187)

Add new sys.unraisablehook() function which can be overridden to
control how "unraisable exceptions" are handled. It is called when an
exception has occurred but there is no way for Python to handle it.
For example, when a destructor raises an exception or during garbage
collection (gc.collect()).

Changes:

* Add an internal UnraisableHookArgs type used to pass arguments to
  sys.unraisablehook.
* Add _PyErr_WriteUnraisableDefaultHook().
* The default hook now ignores exception on writing the traceback.
* test_sys now uses unittest.main() to automatically discover tests:
  remove test_main().
* Add _PyErr_Init().
* Fix PyErr_WriteUnraisable(): hold a strong reference to sys.stderr
  while using it

Author: vstinner

Doc/c-api/exceptions.rst

@@ -72,6 +72,9 @@ Printing and clearing
 
 .. c:function:: void PyErr_WriteUnraisable(PyObject *obj)
 
+   Call :func:`sys.unraisablehook` using the current exception and *obj*
+   argument.
+
    This utility function prints a warning message to ``sys.stderr`` when an
    exception has been set but it is impossible for the interpreter to actually
    raise the exception.  It is used, for example, when an exception occurs in an
@@ -81,6 +84,8 @@ Printing and clearing
    in which the unraisable exception occurred. If possible,
    the repr of *obj* will be printed in the warning message.
 
+   An exception must be set when calling this function.
+
 
 Raising exceptions
 ==================

Doc/library/sys.rst

@@ -248,16 +248,19 @@ always available.
    before the program exits.  The handling of such top-level exceptions can be
    customized by assigning another three-argument function to ``sys.excepthook``.
 
+   See also :func:`unraisablehook` which handles unraisable exceptions.
+
 
 .. data:: __breakpointhook__
           __displayhook__
           __excepthook__
+          __unraisablehook__
 
    These objects contain the original values of ``breakpointhook``,
-   ``displayhook``, and ``excepthook`` at the start of the program.  They are
-   saved so that ``breakpointhook``, ``displayhook`` and ``excepthook`` can be
-   restored in case they happen to get replaced with broken or alternative
-   objects.
+   ``displayhook``, ``excepthook``, and ``unraisablehook`` at the start of the
+   program.  They are saved so that ``breakpointhook``, ``displayhook`` and
+   ``excepthook``, ``unraisablehook`` can be restored in case they happen to
+   get replaced with broken or alternative objects.
 
    .. versionadded:: 3.7
       __breakpointhook__
@@ -1487,6 +1490,28 @@ always available.
    is suppressed and only the exception type and value are printed.
 
 
+.. function:: unraisablehook(unraisable, /)
+
+   Handle an unraisable exception.
+
+   Called when an exception has occurred but there is no way for Python to
+   handle it. For example, when a destructor raises an exception or during
+   garbage collection (:func:`gc.collect`).
+
+   The *unraisable* argument has the following attributes:
+
+   * *exc_type*: Exception type.
+   * *exc_value*: Exception value, can be ``None``.
+   * *exc_traceback*: Exception traceback, can be ``None``.
+   * *object*: Object causing the exception, can be ``None``.
+
+   :func:`sys.unraisablehook` can be overridden to control how unraisable
+   exceptions are handled.
+
+   See also :func:`excepthook` which handles uncaught exceptions.
+
+   .. versionadded:: 3.8
+
 .. data:: version
 
    A string containing the version number of the Python interpreter plus additional

Doc/whatsnew/3.8.rst

@@ -481,6 +481,16 @@ and manipulating normal distributions of a random variable.
     [7.672102882379219, 12.000027119750287, 4.647488369766392]
 
 
+sys
+---
+
+Add new :func:`sys.unraisablehook` function which can be overridden to control
+how "unraisable exceptions" are handled. It is called when an exception has
+occurred but there is no way for Python to handle it. For example, when a
+destructor raises an exception or during garbage collection
+(:func:`gc.collect`).
+
+
 tarfile
 -------
 

Include/internal/pycore_pylifecycle.h

@@ -48,6 +48,7 @@ extern int _PySys_InitMain(
     PyInterpreterState *interp);
 extern _PyInitError _PyImport_Init(PyInterpreterState *interp);
 extern _PyInitError _PyExc_Init(void);
+extern _PyInitError _PyErr_Init(void);
 extern _PyInitError _PyBuiltins_AddExceptions(PyObject * bltinmod);
 extern _PyInitError _PyImportHooks_Init(void);
 extern int _PyFloat_Init(void);
@@ -100,8 +101,11 @@ PyAPI_FUNC(_PyInitError) _Py_PreInitializeFromCoreConfig(
     const _PyCoreConfig *coreconfig,
     const _PyArgv *args);
 
+
 PyAPI_FUNC(int) _Py_HandleSystemExit(int *exitcode_p);
 
+PyAPI_FUNC(PyObject*) _PyErr_WriteUnraisableDefaultHook(PyObject *unraisable);
+
 #ifdef __cplusplus
 }
 #endif

Lib/test/test_sys.py

@@ -876,6 +876,81 @@ def test__enablelegacywindowsfsencoding(self):
         self.assertEqual(out, 'mbcs replace')
 
 
+@test.support.cpython_only
+class UnraisableHookTest(unittest.TestCase):
+    def write_unraisable_exc(self, exc, obj):
+        import _testcapi
+        import types
+        try:
+            # raise the exception to get a traceback in the except block
+            try:
+                raise exc
+            except Exception as exc2:
+                _testcapi.write_unraisable_exc(exc2, obj)
+                return types.SimpleNamespace(exc_type=type(exc2),
+                                             exc_value=exc2,
+                                             exc_traceback=exc2.__traceback__,
+                                             object=obj)
+        finally:
+            # Explicitly break any reference cycle
+            exc = None
+            exc2 = None
+
+    def test_original_unraisablehook(self):
+        obj = "an object"
+
+        with test.support.captured_output("stderr") as stderr:
+            with test.support.swap_attr(sys, 'unraisablehook',
+                                        sys.__unraisablehook__):
+                self.write_unraisable_exc(ValueError(42), obj)
+
+        err = stderr.getvalue()
+        self.assertIn(f'Exception ignored in: {obj!r}\n', err)
+        self.assertIn('Traceback (most recent call last):\n', err)
+        self.assertIn('ValueError: 42\n', err)
+
+    def test_original_unraisablehook_wrong_type(self):
+        exc = ValueError(42)
+        with test.support.swap_attr(sys, 'unraisablehook',
+                                    sys.__unraisablehook__):
+            with self.assertRaises(TypeError):
+                sys.unraisablehook(exc)
+
+    def test_custom_unraisablehook(self):
+        hook_args = None
+
+        def hook_func(args):
+            nonlocal hook_args
+            hook_args = args
+
+        obj = object()
+        try:
+            with test.support.swap_attr(sys, 'unraisablehook', hook_func):
+                expected = self.write_unraisable_exc(ValueError(42), obj)
+                for attr in "exc_type exc_value exc_traceback object".split():
+                    self.assertEqual(getattr(hook_args, attr),
+                                     getattr(expected, attr),
+                                     (hook_args, expected))
+        finally:
+            # expected and hook_args contain an exception: break reference cycle
+            expected = None
+            hook_args = None
+
+    def test_custom_unraisablehook_fail(self):
+        def hook_func(*args):
+            raise Exception("hook_func failed")
+
+        with test.support.captured_output("stderr") as stderr:
+            with test.support.swap_attr(sys, 'unraisablehook', hook_func):
+                self.write_unraisable_exc(ValueError(42), None)
+
+        err = stderr.getvalue()
+        self.assertIn(f'Exception ignored in: {hook_func!r}\n',
+                      err)
+        self.assertIn('Traceback (most recent call last):\n', err)
+        self.assertIn('Exception: hook_func failed\n', err)
+
+
 @test.support.cpython_only
 class SizeofTest(unittest.TestCase):
 
@@ -1277,8 +1352,5 @@ def test_asyncgen_hooks(self):
         self.assertIsNone(cur.finalizer)
 
 
-def test_main():
-    test.support.run_unittest(SysModuleTest, SizeofTest)
-
 if __name__ == "__main__":
-    test_main()
+    unittest.main()

Misc/NEWS.d/next/Library/2019-05-08-12-51-37.bpo-36829.8enFMA.rst

@@ -0,0 +1,5 @@
+Add new :func:`sys.unraisablehook` function which can be overridden to
+control how "unraisable exceptions" are handled. It is called when an
+exception has occurred but there is no way for Python to handle it. For
+example, when a destructor raises an exception or during garbage collection
+(:func:`gc.collect`).

Modules/_testcapimodule.c

@@ -4982,6 +4982,20 @@ negative_refcount(PyObject *self, PyObject *Py_UNUSED(args))
 #endif
 
 
+static PyObject*
+test_write_unraisable_exc(PyObject *self, PyObject *args)
+{
+    PyObject *exc, *obj;
+    if (!PyArg_ParseTuple(args, "OO", &exc, &obj)) {
+        return NULL;
+    }
+
+    PyErr_SetObject((PyObject *)Py_TYPE(exc), exc);
+    PyErr_WriteUnraisable(obj);
+    Py_RETURN_NONE;
+}
+
+
 static PyMethodDef TestMethods[] = {
     {"raise_exception",         raise_exception,                 METH_VARARGS},
     {"raise_memoryerror",       raise_memoryerror,               METH_NOARGS},
@@ -5221,6 +5235,7 @@ static PyMethodDef TestMethods[] = {
 #ifdef Py_REF_DEBUG
     {"negative_refcount", negative_refcount, METH_NOARGS},
 #endif
+    {"write_unraisable_exc", test_write_unraisable_exc, METH_VARARGS},
     {NULL, NULL} /* sentinel */
 };