changeset: 100551:cef6a32d805f user: Victor Stinner date: Tue Mar 15 22:22:13 2016 +0100 files: Doc/c-api/memory.rst Doc/whatsnew/3.6.rst Misc/NEWS Modules/_tracemalloc.c Modules/hashtable.c Objects/bytearrayobject.c Objects/obmalloc.c Parser/pgenmain.c description: On memory error, dump the memory block traceback Issue #26564: _PyObject_DebugDumpAddress() now dumps the traceback where a memory block was allocated on memory block. Use the tracemalloc module to get the traceback. diff -r 5eb223e1638c -r cef6a32d805f Doc/c-api/memory.rst --- a/Doc/c-api/memory.rst Tue Mar 15 21:57:23 2016 +0100 +++ b/Doc/c-api/memory.rst Tue Mar 15 22:22:13 2016 +0100 @@ -349,12 +349,19 @@ allocator functions of the :c:data:`PYMEM_DOMAIN_OBJ` domain (ex: :c:func:`PyObject_Malloc`) are called + On error, the debug hooks use the :mod:`tracemalloc` module to get the + traceback where a memory block was allocated. The traceback is only + displayed if :mod:`tracemalloc` is tracing Python memory allocations and the + memory block was traced. + These hooks are installed by default if Python is compiled in debug mode. The :envvar:`PYTHONMALLOC` environment variable can be used to install debug hooks on a Python compiled in release mode. .. versionchanged:: 3.6 This function now also works on Python compiled in release mode. + On error, the debug hooks now use :mod:`tracemalloc` to get the traceback + where a memory block was allocated. .. _pymalloc: diff -r 5eb223e1638c -r cef6a32d805f Doc/whatsnew/3.6.rst --- a/Doc/whatsnew/3.6.rst Tue Mar 15 21:57:23 2016 +0100 +++ b/Doc/whatsnew/3.6.rst Tue Mar 15 22:22:13 2016 +0100 @@ -129,7 +129,48 @@ It helps to use external memory debuggers like Valgrind on a Python compiled in release mode. -(Contributed by Victor Stinner in :issue:`26516`.) +On error, the debug hooks on Python memory allocators now use the +:mod:`tracemalloc` module to get the traceback where a memory block was +allocated. + +Example of fatal error on buffer overflow using +``python3.6 -X tracemalloc=5`` (store 5 frames in traces):: + + Debug memory block at address p=0x7fbcd41666f8: API 'o' + 4 bytes originally requested + The 7 pad bytes at p-7 are FORBIDDENBYTE, as expected. + The 8 pad bytes at tail=0x7fbcd41666fc are not all FORBIDDENBYTE (0xfb): + at tail+0: 0x02 *** OUCH + at tail+1: 0xfb + at tail+2: 0xfb + at tail+3: 0xfb + at tail+4: 0xfb + at tail+5: 0xfb + at tail+6: 0xfb + at tail+7: 0xfb + The block was made by call #1233329 to debug malloc/realloc. + Data at p: 1a 2b 30 00 + + Memory block allocated at (most recent call first): + File "test/test_bytes.py", line 323 + File "unittest/case.py", line 600 + File "unittest/case.py", line 648 + File "unittest/suite.py", line 122 + File "unittest/suite.py", line 84 + + Fatal Python error: bad trailing pad byte + + Current thread 0x00007fbcdbd32700 (most recent call first): + File "test/test_bytes.py", line 323 in test_hex + File "unittest/case.py", line 600 in run + File "unittest/case.py", line 648 in __call__ + File "unittest/suite.py", line 122 in run + File "unittest/suite.py", line 84 in __call__ + File "unittest/suite.py", line 122 in run + File "unittest/suite.py", line 84 in __call__ + ... + +(Contributed by Victor Stinner in :issue:`26516` and :issue:`26564`.) Other Language Changes diff -r 5eb223e1638c -r cef6a32d805f Misc/NEWS --- a/Misc/NEWS Tue Mar 15 21:57:23 2016 +0100 +++ b/Misc/NEWS Tue Mar 15 22:22:13 2016 +0100 @@ -10,6 +10,10 @@ Core and Builtins ----------------- +- Issue #26564: On error, the debug hooks on Python memory allocators now use + the :mod:`tracemalloc` module to get the traceback where a memory block was + allocated. + - Issue #26558: The debug hooks on Python memory allocator :c:func:`PyObject_Malloc` now detect when functions are called without holding the GIL. diff -r 5eb223e1638c -r cef6a32d805f Modules/_tracemalloc.c --- a/Modules/_tracemalloc.c Tue Mar 15 21:57:23 2016 +0100 +++ b/Modules/_tracemalloc.c Tue Mar 15 22:22:13 2016 +0100 @@ -1161,6 +1161,25 @@ return get_traces.list; } +static traceback_t* +tracemalloc_get_traceback(const void *ptr) +{ + trace_t trace; + int found; + + if (!tracemalloc_config.tracing) + return NULL; + + TABLES_LOCK(); + found = _Py_HASHTABLE_GET(tracemalloc_traces, ptr, trace); + TABLES_UNLOCK(); + + if (!found) + return NULL; + + return trace.traceback; +} + PyDoc_STRVAR(tracemalloc_get_object_traceback_doc, "_get_object_traceback(obj)\n" "\n" @@ -1175,11 +1194,7 @@ { PyTypeObject *type; void *ptr; - trace_t trace; - int found; - - if (!tracemalloc_config.tracing) - Py_RETURN_NONE; + traceback_t *traceback; type = Py_TYPE(obj); if (PyType_IS_GC(type)) @@ -1187,16 +1202,46 @@ else ptr = (void *)obj; - TABLES_LOCK(); - found = _Py_HASHTABLE_GET(tracemalloc_traces, ptr, trace); - TABLES_UNLOCK(); - - if (!found) + traceback = tracemalloc_get_traceback(ptr); + if (traceback == NULL) Py_RETURN_NONE; - return traceback_to_pyobject(trace.traceback, NULL); + return traceback_to_pyobject(traceback, NULL); +} + +#define PUTS(fd, str) _Py_write_noraise(fd, str, (int)strlen(str)) + +static void +_PyMem_DumpFrame(int fd, frame_t * frame) +{ + PUTS(fd, " File \""); + _Py_DumpASCII(fd, frame->filename); + PUTS(fd, "\", line "); + _Py_DumpDecimal(fd, frame->lineno); + PUTS(fd, "\n"); } +/* Dump the traceback where a memory block was allocated into file descriptor + fd. The function may block on TABLES_LOCK() but it is unlikely. */ +void +_PyMem_DumpTraceback(int fd, const void *ptr) +{ + traceback_t *traceback; + int i; + + traceback = tracemalloc_get_traceback(ptr); + if (traceback == NULL) + return; + + PUTS(fd, "Memory block allocated at (most recent call first):\n"); + for (i=0; i < traceback->nframe; i++) { + _PyMem_DumpFrame(fd, &traceback->frames[i]); + } + PUTS(fd, "\n"); +} + +#undef PUTS + PyDoc_STRVAR(tracemalloc_start_doc, "start(nframe: int=1)\n" "\n" diff -r 5eb223e1638c -r cef6a32d805f Modules/hashtable.c --- a/Modules/hashtable.c Tue Mar 15 21:57:23 2016 +0100 +++ b/Modules/hashtable.c Tue Mar 15 22:22:13 2016 +0100 @@ -486,9 +486,9 @@ void *data, *new_data; dst = _Py_hashtable_new_full(src->data_size, src->num_buckets, - src->hash_func, src->compare_func, - src->copy_data_func, src->free_data_func, - src->get_data_size_func, &src->alloc); + src->hash_func, src->compare_func, + src->copy_data_func, src->free_data_func, + src->get_data_size_func, &src->alloc); if (dst == NULL) return NULL; diff -r 5eb223e1638c -r cef6a32d805f Objects/bytearrayobject.c --- a/Objects/bytearrayobject.c Tue Mar 15 21:57:23 2016 +0100 +++ b/Objects/bytearrayobject.c Tue Mar 15 22:22:13 2016 +0100 @@ -2820,6 +2820,7 @@ { char* argbuf = PyByteArray_AS_STRING(self); Py_ssize_t arglen = PyByteArray_GET_SIZE(self); + PyByteArray_AS_STRING(self)[arglen+1] = 2; return _Py_strhex(argbuf, arglen); } diff -r 5eb223e1638c -r cef6a32d805f Objects/obmalloc.c --- a/Objects/obmalloc.c Tue Mar 15 21:57:23 2016 +0100 +++ b/Objects/obmalloc.c Tue Mar 15 22:22:13 2016 +0100 @@ -1,5 +1,10 @@ #include "Python.h" + +/* Defined in tracemalloc.c */ +extern void _PyMem_DumpTraceback(int fd, const void *ptr); + + /* Python's malloc wrappers (see pymem.h) */ /* @@ -2202,6 +2207,10 @@ } fputc('\n', stderr); } + fputc('\n', stderr); + + fflush(stderr); + _PyMem_DumpTraceback(fileno(stderr), p); } diff -r 5eb223e1638c -r cef6a32d805f Parser/pgenmain.c --- a/Parser/pgenmain.c Tue Mar 15 21:57:23 2016 +0100 +++ b/Parser/pgenmain.c Tue Mar 15 22:22:13 2016 +0100 @@ -38,11 +38,11 @@ } #ifdef WITH_THREAD -/* Needed by obmalloc.c */ +/* Functions needed by obmalloc.c */ int PyGILState_Check(void) -{ - return 1; -} +{ return 1; } +void _PyMem_DumpTraceback(int fd, const void *ptr) +{} #endif int