bpo-32043: New "developer mode": "-X dev" option (#4413)

Add a new "developer mode": new "-X dev" command line option to
enable debug checks at runtime.

Changes:

* Add unit tests for -X dev
* test_cmd_line: replace test.support with support.
* Fix _PyRuntimeState_Fini(): Use the same memory allocator
   than _PyRuntimeState_Init().
* Fix _PyMem_GetDefaultRawAllocator()

Author: vstinner

Doc/using/cmdline.rst

@@ -413,6 +413,17 @@ Miscellaneous options
      nested imports).  Note that its output may be broken in multi-threaded
      application.  Typical usage is ``python3 -X importtime -c 'import
      asyncio'``.  See also :envvar:`PYTHONPROFILEIMPORTTIME`.
+   * ``-X dev`` enables the "developer mode": enable debug checks at runtime.
+     In short, ``python3 -X dev ...`` behaves as ``PYTHONMALLOC=debug python3
+     -W default -X faulthandler ...``, except that the :envvar:`PYTHONMALLOC`
+     environment variable is not set in practice. Developer mode:
+
+     * Add ``default`` warnings option. For example, display
+       :exc:`DeprecationWarning` and :exc:`ResourceWarning` warnings.
+     * Install debug hooks on memory allocators as if :envvar:`PYTHONMALLOC`
+       is set to ``debug``.
+     * Enable the :mod:`faulthandler` module to dump the Python traceback
+       on a crash.
 
    It also allows passing arbitrary values and retrieving them through the
    :data:`sys._xoptions` dictionary.
@@ -430,7 +441,8 @@ Miscellaneous options
       The ``-X showalloccount`` option.
 
    .. versionadded:: 3.7
-      The ``-X importtime`` and :envvar:`PYTHONPROFILEIMPORTTIME` options.
+      The ``-X importtime``, ``-X dev`` and :envvar:`PYTHONPROFILEIMPORTTIME`
+      options.
 
 
 Options you shouldn't use

Doc/whatsnew/3.7.rst

@@ -185,6 +185,19 @@ resolution on Linux and Windows.
        PEP written and implemented by Victor Stinner
 
 
+New Developer Mode: -X dev
+--------------------------
+
+Add a new "developer mode": ``-X dev`` command line option to enable debug
+checks at runtime.
+
+In short, ``python3 -X dev ...`` behaves as ``PYTHONMALLOC=debug python3 -W
+default -X faulthandler ...``, except that the PYTHONMALLOC environment
+variable is not set in practice.
+
+See :option:`-X` ``dev`` for the details.
+
+
 Other Language Changes
 ======================
 

Lib/test/test_cmd_line.py

@@ -2,14 +2,15 @@
 # Most tests are executed with environment variables ignored
 # See test_cmd_line_script.py for testing of script execution
 
-import test.support, unittest
 import os
-import sys
 import subprocess
+import sys
 import tempfile
-from test.support import script_helper, is_android
+import unittest
+from test import support
 from test.support.script_helper import (
-    spawn_python, kill_python, assert_python_ok, assert_python_failure
+    spawn_python, kill_python, assert_python_ok, assert_python_failure,
+    interpreter_requires_environment
 )
 
 # XXX (ncoghlan): Move to script_helper and make consistent with run_python
@@ -132,11 +133,11 @@ def test_run_code(self):
         # All good if execution is successful
         assert_python_ok('-c', 'pass')
 
-    @unittest.skipUnless(test.support.FS_NONASCII, 'need support.FS_NONASCII')
+    @unittest.skipUnless(support.FS_NONASCII, 'need support.FS_NONASCII')
     def test_non_ascii(self):
         # Test handling of non-ascii data
         command = ("assert(ord(%r) == %s)"
-                   % (test.support.FS_NONASCII, ord(test.support.FS_NONASCII)))
+                   % (support.FS_NONASCII, ord(support.FS_NONASCII)))
         assert_python_ok('-c', command)
 
     # On Windows, pass bytes to subprocess doesn't test how Python decodes the
@@ -179,7 +180,7 @@ def test_undecodable_code(self):
             raise AssertionError("%a doesn't start with %a" % (stdout, pattern))
 
     @unittest.skipUnless((sys.platform == 'darwin' or
-                is_android), 'test specific to Mac OS X and Android')
+                support.is_android), 'test specific to Mac OS X and Android')
     def test_osx_android_utf8(self):
         def check_output(text):
             decoded = text.decode('utf-8', 'surrogateescape')
@@ -385,7 +386,7 @@ def preexec():
             stderr=subprocess.PIPE,
             preexec_fn=preexec)
         out, err = p.communicate()
-        self.assertEqual(test.support.strip_python_stderr(err), b'')
+        self.assertEqual(support.strip_python_stderr(err), b'')
         self.assertEqual(p.returncode, 42)
 
     def test_no_stdin(self):
@@ -433,8 +434,8 @@ def test_del___main__(self):
         # Issue #15001: PyRun_SimpleFileExFlags() did crash because it kept a
         # borrowed reference to the dict of __main__ module and later modify
         # the dict whereas the module was destroyed
-        filename = test.support.TESTFN
-        self.addCleanup(test.support.unlink, filename)
+        filename = support.TESTFN
+        self.addCleanup(support.unlink, filename)
         with open(filename, "w") as script:
             print("import sys", file=script)
             print("del sys.modules['__main__']", file=script)
@@ -458,7 +459,7 @@ def test_unknown_options(self):
         self.assertEqual(err.splitlines().count(b'Unknown option: -a'), 1)
         self.assertEqual(b'', out)
 
-    @unittest.skipIf(script_helper.interpreter_requires_environment(),
+    @unittest.skipIf(interpreter_requires_environment(),
                      'Cannot run -I tests when PYTHON env vars are required.')
     def test_isolatedmode(self):
         self.verify_valid_flag('-I')
@@ -469,7 +470,7 @@ def test_isolatedmode(self):
             # dummyvar to prevent extraneous -E
             dummyvar="")
         self.assertEqual(out.strip(), b'1 1 1')
-        with test.support.temp_cwd() as tmpdir:
+        with support.temp_cwd() as tmpdir:
             fake = os.path.join(tmpdir, "uuid.py")
             main = os.path.join(tmpdir, "main.py")
             with open(fake, "w") as f:
@@ -506,6 +507,46 @@ def test_sys_flags_set(self):
             with self.subTest(envar_value=value):
                 assert_python_ok('-c', code, **env_vars)
 
+    def run_xdev(self, code, check_exitcode=True):
+        env = dict(os.environ)
+        env.pop('PYTHONWARNINGS', None)
+        # Force malloc() to disable the debug hooks which are enabled
+        # by default for Python compiled in debug mode
+        env['PYTHONMALLOC'] = 'malloc'
+
+        args = (sys.executable, '-X', 'dev', '-c', code)
+        proc = subprocess.run(args,
+                              stdout=subprocess.PIPE,
+                              stderr=subprocess.STDOUT,
+                              universal_newlines=True,
+                              env=env)
+        if check_exitcode:
+            self.assertEqual(proc.returncode, 0, proc)
+        return proc.stdout.rstrip()
+
+    def test_xdev(self):
+        out = self.run_xdev("import sys; print(sys.warnoptions)")
+        self.assertEqual(out, "['default']")
+
+        try:
+            import _testcapi
+        except ImportError:
+            pass
+        else:
+            code = "import _testcapi; _testcapi.pymem_api_misuse()"
+            with support.SuppressCrashReport():
+                out = self.run_xdev(code, check_exitcode=False)
+            self.assertIn("Debug memory block at address p=", out)
+
+        try:
+            import faulthandler
+        except ImportError:
+            pass
+        else:
+            code = "import faulthandler; print(faulthandler.is_enabled())"
+            out = self.run_xdev(code)
+            self.assertEqual(out, "True")
+
 class IgnoreEnvironmentTest(unittest.TestCase):
 
     def run_ignoring_vars(self, predicate, **env_vars):
@@ -541,8 +582,8 @@ def test_sys_flags_not_set(self):
 
 
 def test_main():
-    test.support.run_unittest(CmdLineTest, IgnoreEnvironmentTest)
-    test.support.reap_children()
+    support.run_unittest(CmdLineTest, IgnoreEnvironmentTest)
+    support.reap_children()
 
 if __name__ == "__main__":
     test_main()

Misc/NEWS.d/next/Core and Builtins/2017-11-16-03-44-08.bpo-32043.AAzwpZ.rst

@@ -0,0 +1,2 @@
+Add a new "developer mode": new "-X dev" command line option to enable debug
+checks at runtime.

Modules/main.c

@@ -1396,6 +1396,16 @@ pymain_parse_envvars(_PyMain *pymain)
     if (pymain_init_tracemalloc(pymain) < 0) {
         return -1;
     }
+    if (pymain_get_xoption(pymain, L"dev")) {
+        /* "python3 -X dev ..." behaves
+           as "PYTHONMALLOC=debug python3 -Wd -X faulthandler ..." */
+        core_config->allocator = "debug";
+        if (pymain_optlist_append(pymain, &pymain->cmdline.warning_options,
+                                  L"default") < 0) {
+            return -1;
+        }
+        core_config->faulthandler = 1;
+    }
     return 0;
 }
 

Objects/obmalloc.c

@@ -190,8 +190,14 @@ static struct {
 void
 _PyMem_GetDefaultRawAllocator(PyMemAllocatorEx *alloc_p)
 {
-    PyMemAllocatorEx alloc = {NULL, PYRAW_FUNCS};
-    *alloc_p = alloc;
+    PyMemAllocatorEx pymem_raw = {
+#ifdef Py_DEBUG
+        &_PyMem_Debug.raw, PYRAWDBG_FUNCS
+#else
+        NULL, PYRAW_FUNCS
+#endif
+    };
+    *alloc_p = pymem_raw;
 }
 
 int
@@ -274,13 +280,6 @@ _PyObject_Initialize(struct _pyobj_runtime_state *state)
 void
 _PyMem_Initialize(struct _pymem_runtime_state *state)
 {
-    PyMemAllocatorEx pymem_raw = {
-#ifdef Py_DEBUG
-        &_PyMem_Debug.raw, PYRAWDBG_FUNCS
-#else
-        NULL, PYRAW_FUNCS
-#endif
-    };
     PyMemAllocatorEx pymem = {
 #ifdef Py_DEBUG
         &_PyMem_Debug.mem, PYDBG_FUNCS
@@ -296,7 +295,7 @@ _PyMem_Initialize(struct _pymem_runtime_state *state)
 #endif
     };
 
-    state->allocators.raw = pymem_raw;
+    _PyMem_GetDefaultRawAllocator(&state->allocators.raw);
     state->allocators.mem = pymem;
     state->allocators.obj = pyobject;
 

Python/pystate.c

@@ -64,10 +64,18 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime)
 void
 _PyRuntimeState_Fini(_PyRuntimeState *runtime)
 {
+    /* Use the same memory allocator than _PyRuntimeState_Init() */
+    PyMemAllocatorEx old_alloc, raw_alloc;
+    PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+    _PyMem_GetDefaultRawAllocator(&raw_alloc);
+    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &raw_alloc);
+
     if (runtime->interpreters.mutex != NULL) {
         PyThread_free_lock(runtime->interpreters.mutex);
         runtime->interpreters.mutex = NULL;
     }
+
+    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 }
 
 #define HEAD_LOCK() PyThread_acquire_lock(_PyRuntime.interpreters.mutex, \