changeset:   100559:58644086c195
user:        Victor Stinner <victor.stinner@gmail.com>
date:        Wed Mar 16 12:12:53 2016 +0100
files:       Lib/test/test_capi.py Misc/NEWS Modules/_testcapimodule.c Objects/obmalloc.c
description:
Fail if PyMem_Malloc() is called without holding the GIL

Issue #26563: Debug hooks on Python memory allocators now raise a fatal error
if functions of the PyMem_Malloc() family are called without holding the GIL.


diff -r 5f2284ecf9c6 -r 58644086c195 Lib/test/test_capi.py
--- a/Lib/test/test_capi.py	Wed Mar 16 09:43:14 2016 +0100
+++ b/Lib/test/test_capi.py	Wed Mar 16 12:12:53 2016 +0100
@@ -602,15 +602,24 @@
         regex = regex.format(ptr=self.PTR_REGEX)
         self.assertRegex(out, regex)
 
-    def test_pyobject_malloc_without_gil(self):
-        # Calling PyObject_Malloc() without holding the GIL must raise an
-        # error in debug mode.
-        code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()'
+    def check_malloc_without_gil(self, code):
         out = self.check(code)
         expected = ('Fatal Python error: Python memory allocator called '
                     'without holding the GIL')
         self.assertIn(expected, out)
 
+    def test_pymem_malloc_without_gil(self):
+        # Debug hooks must raise an error if PyMem_Malloc() is called
+        # without holding the GIL
+        code = 'import _testcapi; _testcapi.pymem_malloc_without_gil()'
+        self.check_malloc_without_gil(code)
+
+    def test_pyobject_malloc_without_gil(self):
+        # Debug hooks must raise an error if PyObject_Malloc() is called
+        # without holding the GIL
+        code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()'
+        self.check_malloc_without_gil(code)
+
 
 class PyMemMallocDebugTests(PyMemDebugTests):
     PYTHONMALLOC = 'malloc_debug'
diff -r 5f2284ecf9c6 -r 58644086c195 Misc/NEWS
--- a/Misc/NEWS	Wed Mar 16 09:43:14 2016 +0100
+++ b/Misc/NEWS	Wed Mar 16 12:12:53 2016 +0100
@@ -10,6 +10,10 @@
 Core and Builtins
 -----------------
 
+- Issue #26563: Debug hooks on Python memory allocators now raise a fatal
+  error if functions of the :c:func:`PyMem_Malloc` family are called without
+  holding the GIL.
+
 - Issue #26564: On error, the debug hooks on Python memory allocators now use
   the :mod:`tracemalloc` module to get the traceback where a memory block was
   allocated.
diff -r 5f2284ecf9c6 -r 58644086c195 Modules/_testcapimodule.c
--- a/Modules/_testcapimodule.c	Wed Mar 16 09:43:14 2016 +0100
+++ b/Modules/_testcapimodule.c	Wed Mar 16 12:12:53 2016 +0100
@@ -3644,10 +3644,28 @@
 }
 
 static PyObject*
+pymem_malloc_without_gil(PyObject *self, PyObject *args)
+{
+    char *buffer;
+
+    /* Deliberate bug to test debug hooks on Python memory allocators:
+       call PyMem_Malloc() without holding the GIL */
+    Py_BEGIN_ALLOW_THREADS
+    buffer = PyMem_Malloc(10);
+    Py_END_ALLOW_THREADS
+
+    PyMem_Free(buffer);
+
+    Py_RETURN_NONE;
+}
+
+static PyObject*
 pyobject_malloc_without_gil(PyObject *self, PyObject *args)
 {
     char *buffer;
 
+    /* Deliberate bug to test debug hooks on Python memory allocators:
+       call PyObject_Malloc() without holding the GIL */
     Py_BEGIN_ALLOW_THREADS
     buffer = PyObject_Malloc(10);
     Py_END_ALLOW_THREADS
@@ -3841,6 +3859,7 @@
     {"get_recursion_depth", get_recursion_depth, METH_NOARGS},
     {"pymem_buffer_overflow", pymem_buffer_overflow, METH_NOARGS},
     {"pymem_api_misuse", pymem_api_misuse, METH_NOARGS},
+    {"pymem_malloc_without_gil", pymem_malloc_without_gil, METH_NOARGS},
     {"pyobject_malloc_without_gil", pyobject_malloc_without_gil, METH_NOARGS},
     {NULL, NULL} /* sentinel */
 };
diff -r 5f2284ecf9c6 -r 58644086c195 Objects/obmalloc.c
--- a/Objects/obmalloc.c	Wed Mar 16 09:43:14 2016 +0100
+++ b/Objects/obmalloc.c	Wed Mar 16 12:12:53 2016 +0100
@@ -198,7 +198,7 @@
 
 static PyMemAllocatorEx _PyMem = {
 #ifdef Py_DEBUG
-    &_PyMem_Debug.mem, PYRAWDBG_FUNCS
+    &_PyMem_Debug.mem, PYDBG_FUNCS
 #else
     NULL, PYMEM_FUNCS
 #endif
@@ -321,17 +321,17 @@
         PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc);
     }
 
-    if (_PyMem.malloc != _PyMem_DebugRawMalloc) {
+    alloc.malloc = _PyMem_DebugMalloc;
+    alloc.calloc = _PyMem_DebugCalloc;
+    alloc.realloc = _PyMem_DebugRealloc;
+    alloc.free = _PyMem_DebugFree;
+
+    if (_PyMem.malloc != _PyMem_DebugMalloc) {
         alloc.ctx = &_PyMem_Debug.mem;
         PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &_PyMem_Debug.mem.alloc);
         PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc);
     }
 
-    alloc.malloc = _PyMem_DebugMalloc;
-    alloc.calloc = _PyMem_DebugCalloc;
-    alloc.realloc = _PyMem_DebugRealloc;
-    alloc.free = _PyMem_DebugFree;
-
     if (_PyObject.malloc != _PyMem_DebugMalloc) {
         alloc.ctx = &_PyMem_Debug.obj;
         PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &_PyMem_Debug.obj.alloc);