changeset: 100598:2428d794b0e1 user: Victor Stinner date: Sat Mar 19 01:03:51 2016 +0100 files: Doc/c-api/exceptions.rst Doc/library/warnings.rst Doc/whatsnew/3.6.rst Include/warnings.h Lib/test/test_warnings/__init__.py Lib/warnings.py Misc/NEWS Modules/_io/fileio.c Modules/posixmodule.c Modules/socketmodule.c Python/_warnings.c description: On ResourceWarning, log traceback where the object was allocated Issue #26567: * Add a new function PyErr_ResourceWarning() function to pass the destroyed object * Add a source attribute to warnings.WarningMessage * Add warnings._showwarnmsg() which uses tracemalloc to get the traceback where source object was allocated. diff -r 79888b970cc4 -r 2428d794b0e1 Doc/c-api/exceptions.rst --- a/Doc/c-api/exceptions.rst Sat Mar 19 00:47:17 2016 +0100 +++ b/Doc/c-api/exceptions.rst Sat Mar 19 01:03:51 2016 +0100 @@ -334,6 +334,14 @@ .. versionadded:: 3.2 +.. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) + + Function similar to :c:func:`PyErr_WarnFormat`, but *category* is + :exc:`ResourceWarning` and pass *source* to :func:`warnings.WarningMessage`. + + .. versionadded:: 3.6 + + Querying the error indicator ============================ diff -r 79888b970cc4 -r 2428d794b0e1 Doc/library/warnings.rst --- a/Doc/library/warnings.rst Sat Mar 19 00:47:17 2016 +0100 +++ b/Doc/library/warnings.rst Sat Mar 19 01:03:51 2016 +0100 @@ -319,7 +319,7 @@ of the warning message). -.. function:: warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None) +.. function:: warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None, source=None) This is a low-level interface to the functionality of :func:`warn`, passing in explicitly the message, category, filename and line number, and optionally the @@ -335,6 +335,12 @@ source for modules found in zipfiles or other non-filesystem import sources). + *source*, if supplied, is the destroyed object which emitted a + :exc:`ResourceWarning`. + + .. versionchanged:: 3.6 + Add the *source* parameter. + .. function:: showwarning(message, category, filename, lineno, file=None, line=None) diff -r 79888b970cc4 -r 2428d794b0e1 Doc/whatsnew/3.6.rst --- a/Doc/whatsnew/3.6.rst Sat Mar 19 00:47:17 2016 +0100 +++ b/Doc/whatsnew/3.6.rst Sat Mar 19 01:03:51 2016 +0100 @@ -258,6 +258,40 @@ (Contributed by Nikolay Bogoychev in :issue:`16099`.) +warnings +-------- + +A new optional *source* parameter has been added to the +:func:`warnings.warn_explicit` function: the destroyed object which emitted a +:exc:`ResourceWarning`. A *source* attribute has also been added to +:class:`warnings.WarningMessage` (contributed by Victor Stinner in +:issue:`26568` and :issue:`26567`). + +When a :exc:`ResourceWarning` warning is logged, the :mod:`tracemalloc` is now +used to try to retrieve the traceback where the detroyed object was allocated. + +Example with the script ``example.py``:: + + def func(): + f = open(__file__) + f = None + + func() + +Output of the command ``python3.6 -Wd -X tracemalloc=5 example.py``:: + + example.py:3: ResourceWarning: unclosed file <...> + f = None + Object allocated at (most recent call first): + File "example.py", lineno 2 + f = open(__file__) + File "example.py", lineno 5 + func() + +The "Object allocated at" traceback is new and only displayed if +:mod:`tracemalloc` is tracing Python memory allocations. + + zipfile ------- diff -r 79888b970cc4 -r 2428d794b0e1 Include/warnings.h --- a/Include/warnings.h Sat Mar 19 00:47:17 2016 +0100 +++ b/Include/warnings.h Sat Mar 19 01:03:51 2016 +0100 @@ -17,6 +17,13 @@ Py_ssize_t stack_level, const char *format, /* ASCII-encoded string */ ...); + +/* Emit a ResourceWarning warning */ +PyAPI_FUNC(int) PyErr_ResourceWarning( + PyObject *source, + Py_ssize_t stack_level, + const char *format, /* ASCII-encoded string */ + ...); #ifndef Py_LIMITED_API PyAPI_FUNC(int) PyErr_WarnExplicitObject( PyObject *category, diff -r 79888b970cc4 -r 2428d794b0e1 Lib/test/test_warnings/__init__.py --- a/Lib/test/test_warnings/__init__.py Sat Mar 19 00:47:17 2016 +0100 +++ b/Lib/test/test_warnings/__init__.py Sat Mar 19 01:03:51 2016 +0100 @@ -2,7 +2,10 @@ import linecache import os from io import StringIO +import re import sys +import tempfile +import textwrap import unittest from test import support from test.support.script_helper import assert_python_ok, assert_python_failure @@ -763,12 +766,39 @@ file_object, expected_file_line) self.assertEqual(expect, file_object.getvalue()) + class CWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): module = c_warnings class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): module = py_warnings + def test_tracemalloc(self): + with tempfile.NamedTemporaryFile("w", suffix=".py") as tmpfile: + tmpfile.write(textwrap.dedent(""" + def func(): + f = open(__file__) + # Emit ResourceWarning + f = None + + func() + """)) + tmpfile.flush() + fname = tmpfile.name + res = assert_python_ok('-Wd', '-X', 'tracemalloc=2', fname) + stderr = res.err.decode('ascii', 'replace') + stderr = re.sub('<.*>', '<...>', stderr) + expected = textwrap.dedent(f''' + {fname}:5: ResourceWarning: unclosed file <...> + f = None + Object allocated at (most recent call first): + File "{fname}", lineno 3 + f = open(__file__) + File "{fname}", lineno 7 + func() + ''').strip() + self.assertEqual(stderr, expected) + class CatchWarningTests(BaseTest): diff -r 79888b970cc4 -r 2428d794b0e1 Lib/warnings.py --- a/Lib/warnings.py Sat Mar 19 00:47:17 2016 +0100 +++ b/Lib/warnings.py Sat Mar 19 01:03:51 2016 +0100 @@ -2,6 +2,7 @@ import sys + __all__ = ["warn", "warn_explicit", "showwarning", "formatwarning", "filterwarnings", "simplefilter", "resetwarnings", "catch_warnings"] @@ -66,6 +67,18 @@ if line: line = line.strip() s += " %s\n" % line + if msg.source is not None: + import tracemalloc + tb = tracemalloc.get_object_traceback(msg.source) + if tb is not None: + s += 'Object allocated at (most recent call first):\n' + for frame in tb: + s += (' File "%s", lineno %s\n' + % (frame.filename, frame.lineno)) + line = linecache.getline(frame.filename, frame.lineno) + if line: + line = line.strip() + s += ' %s\n' % line return s def filterwarnings(action, message="", category=Warning, module="", lineno=0, @@ -267,7 +280,8 @@ globals) def warn_explicit(message, category, filename, lineno, - module=None, registry=None, module_globals=None): + module=None, registry=None, module_globals=None, + source=None): lineno = int(lineno) if module is None: module = filename or "" @@ -333,17 +347,17 @@ "Unrecognized action (%r) in warnings.filters:\n %s" % (action, item)) # Print message and context - msg = WarningMessage(message, category, filename, lineno) + msg = WarningMessage(message, category, filename, lineno, source) _showwarnmsg(msg) class WarningMessage(object): _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file", - "line") + "line", "source") def __init__(self, message, category, filename, lineno, file=None, - line=None): + line=None, source=None): local_values = locals() for attr in self._WARNING_DETAILS: setattr(self, attr, local_values[attr]) diff -r 79888b970cc4 -r 2428d794b0e1 Misc/NEWS --- a/Misc/NEWS Sat Mar 19 00:47:17 2016 +0100 +++ b/Misc/NEWS Sat Mar 19 01:03:51 2016 +0100 @@ -226,6 +226,11 @@ Library ------- +- Issue #26567: Add a new function :c:func:`PyErr_ResourceWarning` function to + pass the destroyed object. Add a *source* attribute to + :class:`warnings.WarningMessage`. Add warnings._showwarnmsg() which uses + tracemalloc to get the traceback where source object was allocated. + - Issue #26313: ssl.py _load_windows_store_certs fails if windows cert store is empty. Patch by Baji. diff -r 79888b970cc4 -r 2428d794b0e1 Modules/_io/fileio.c --- a/Modules/_io/fileio.c Sat Mar 19 00:47:17 2016 +0100 +++ b/Modules/_io/fileio.c Sat Mar 19 01:03:51 2016 +0100 @@ -92,8 +92,7 @@ if (self->fd >= 0 && self->closefd) { PyObject *exc, *val, *tb; PyErr_Fetch(&exc, &val, &tb); - if (PyErr_WarnFormat(PyExc_ResourceWarning, 1, - "unclosed file %R", source)) { + if (PyErr_ResourceWarning(source, 1, "unclosed file %R", source)) { /* Spurious errors can appear at shutdown */ if (PyErr_ExceptionMatches(PyExc_Warning)) PyErr_WriteUnraisable((PyObject *) self); diff -r 79888b970cc4 -r 2428d794b0e1 Modules/posixmodule.c --- a/Modules/posixmodule.c Sat Mar 19 00:47:17 2016 +0100 +++ b/Modules/posixmodule.c Sat Mar 19 01:03:51 2016 +0100 @@ -12111,8 +12111,8 @@ */ ++Py_REFCNT(iterator); PyErr_Fetch(&exc, &val, &tb); - if (PyErr_WarnFormat(PyExc_ResourceWarning, 1, - "unclosed scandir iterator %R", iterator)) { + if (PyErr_ResourceWarning((PyObject *)iterator, 1, + "unclosed scandir iterator %R", iterator)) { /* Spurious errors can appear at shutdown */ if (PyErr_ExceptionMatches(PyExc_Warning)) PyErr_WriteUnraisable((PyObject *) iterator); diff -r 79888b970cc4 -r 2428d794b0e1 Modules/socketmodule.c --- a/Modules/socketmodule.c Sat Mar 19 00:47:17 2016 +0100 +++ b/Modules/socketmodule.c Sat Mar 19 01:03:51 2016 +0100 @@ -4170,8 +4170,7 @@ Py_ssize_t old_refcount = Py_REFCNT(s); ++Py_REFCNT(s); PyErr_Fetch(&exc, &val, &tb); - if (PyErr_WarnFormat(PyExc_ResourceWarning, 1, - "unclosed %R", s)) + if (PyErr_ResourceWarning(s, 1, "unclosed %R", s)) /* Spurious errors can appear at shutdown */ if (PyErr_ExceptionMatches(PyExc_Warning)) PyErr_WriteUnraisable((PyObject *) s); diff -r 79888b970cc4 -r 2428d794b0e1 Python/_warnings.c --- a/Python/_warnings.c Sat Mar 19 00:47:17 2016 +0100 +++ b/Python/_warnings.c Sat Mar 19 01:03:51 2016 +0100 @@ -287,8 +287,8 @@ } static void -show_warning(PyObject *filename, int lineno, PyObject *text, PyObject - *category, PyObject *sourceline) +show_warning(PyObject *filename, int lineno, PyObject *text, + PyObject *category, PyObject *sourceline) { PyObject *f_stderr; PyObject *name; @@ -362,7 +362,7 @@ static int call_show_warning(PyObject *category, PyObject *text, PyObject *message, PyObject *filename, int lineno, PyObject *lineno_obj, - PyObject *sourceline) + PyObject *sourceline, PyObject *source) { PyObject *show_fn, *msg, *res, *warnmsg_cls = NULL; @@ -388,7 +388,7 @@ } msg = PyObject_CallFunctionObjArgs(warnmsg_cls, message, category, - filename, lineno_obj, + filename, lineno_obj, Py_None, Py_None, source, NULL); Py_DECREF(warnmsg_cls); if (msg == NULL) @@ -412,7 +412,8 @@ static PyObject * warn_explicit(PyObject *category, PyObject *message, PyObject *filename, int lineno, - PyObject *module, PyObject *registry, PyObject *sourceline) + PyObject *module, PyObject *registry, PyObject *sourceline, + PyObject *source) { PyObject *key = NULL, *text = NULL, *result = NULL, *lineno_obj = NULL; PyObject *item = NULL; @@ -521,7 +522,7 @@ goto return_none; if (rc == 0) { if (call_show_warning(category, text, message, filename, lineno, - lineno_obj, sourceline) < 0) + lineno_obj, sourceline, source) < 0) goto cleanup; } else /* if (rc == -1) */ @@ -766,7 +767,8 @@ } static PyObject * -do_warn(PyObject *message, PyObject *category, Py_ssize_t stack_level) +do_warn(PyObject *message, PyObject *category, Py_ssize_t stack_level, + PyObject *source) { PyObject *filename, *module, *registry, *res; int lineno; @@ -775,7 +777,7 @@ return NULL; res = warn_explicit(category, message, filename, lineno, module, registry, - NULL); + NULL, source); Py_DECREF(filename); Py_DECREF(registry); Py_DECREF(module); @@ -796,14 +798,15 @@ category = get_category(message, category); if (category == NULL) return NULL; - return do_warn(message, category, stack_level); + return do_warn(message, category, stack_level, NULL); } static PyObject * warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwd_list[] = {"message", "category", "filename", "lineno", - "module", "registry", "module_globals", 0}; + "module", "registry", "module_globals", + "source", 0}; PyObject *message; PyObject *category; PyObject *filename; @@ -811,10 +814,11 @@ PyObject *module = NULL; PyObject *registry = NULL; PyObject *module_globals = NULL; + PyObject *sourceobj = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOUi|OOO:warn_explicit", + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOUi|OOOO:warn_explicit", kwd_list, &message, &category, &filename, &lineno, &module, - ®istry, &module_globals)) + ®istry, &module_globals, &sourceobj)) return NULL; if (module_globals) { @@ -870,14 +874,14 @@ /* Handle the warning. */ returned = warn_explicit(category, message, filename, lineno, module, - registry, source_line); + registry, source_line, sourceobj); Py_DECREF(source_list); return returned; } standard_call: return warn_explicit(category, message, filename, lineno, module, - registry, NULL); + registry, NULL, sourceobj); } static PyObject * @@ -892,14 +896,14 @@ static int warn_unicode(PyObject *category, PyObject *message, - Py_ssize_t stack_level) + Py_ssize_t stack_level, PyObject *source) { PyObject *res; if (category == NULL) category = PyExc_RuntimeWarning; - res = do_warn(message, category, stack_level); + res = do_warn(message, category, stack_level, source); if (res == NULL) return -1; Py_DECREF(res); @@ -907,12 +911,28 @@ return 0; } +static int +_PyErr_WarnFormatV(PyObject *source, + PyObject *category, Py_ssize_t stack_level, + const char *format, va_list vargs) +{ + PyObject *message; + int res; + + message = PyUnicode_FromFormatV(format, vargs); + if (message == NULL) + return -1; + + res = warn_unicode(category, message, stack_level, source); + Py_DECREF(message); + return res; +} + int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) { - int ret; - PyObject *message; + int res; va_list vargs; #ifdef HAVE_STDARG_PROTOTYPES @@ -920,25 +940,38 @@ #else va_start(vargs); #endif - message = PyUnicode_FromFormatV(format, vargs); - if (message != NULL) { - ret = warn_unicode(category, message, stack_level); - Py_DECREF(message); - } - else - ret = -1; + res = _PyErr_WarnFormatV(NULL, category, stack_level, format, vargs); va_end(vargs); - return ret; + return res; } int +PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, + const char *format, ...) +{ + int res; + va_list vargs; + +#ifdef HAVE_STDARG_PROTOTYPES + va_start(vargs, format); +#else + va_start(vargs); +#endif + res = _PyErr_WarnFormatV(source, PyExc_ResourceWarning, + stack_level, format, vargs); + va_end(vargs); + return res; +} + + +int PyErr_WarnEx(PyObject *category, const char *text, Py_ssize_t stack_level) { int ret; PyObject *message = PyUnicode_FromString(text); if (message == NULL) return -1; - ret = warn_unicode(category, message, stack_level); + ret = warn_unicode(category, message, stack_level, NULL); Py_DECREF(message); return ret; } @@ -964,7 +997,7 @@ if (category == NULL) category = PyExc_RuntimeWarning; res = warn_explicit(category, message, filename, lineno, - module, registry, NULL); + module, registry, NULL, NULL); if (res == NULL) return -1; Py_DECREF(res); @@ -1028,7 +1061,7 @@ if (message != NULL) { PyObject *res; res = warn_explicit(category, message, filename, lineno, - module, registry, NULL); + module, registry, NULL, NULL); Py_DECREF(message); if (res != NULL) { Py_DECREF(res);