bpo-32096: Fall back to defaults in PyMem_RawMalloc() and PyMem_RawFree().

Lib/test/test_capi.py

@@ -593,6 +593,16 @@ def test_forced_io_encoding(self):
         self.maxDiff = None
         self.assertEqual(out.strip(), expected_output)
 
+    def test_pre_initialization_api(self):
+        """
+        Checks the few parts of the C-API that work before the runtine
+        is initialized (via Py_Initialize()).
+        """
+        env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path))
+        out, err = self.run_embedded_interpreter("pre_initialization_api", env=env)
+        self.assertEqual(out, '')
+        self.assertEqual(err, '')
+
 
 class SkipitemTest(unittest.TestCase):
 

Misc/NEWS.d/next/Core and Builtins/2017-11-21-12-44-51.bpo-32096.tXX68e.rst

@@ -0,0 +1,6 @@
+Since the recent global runtime state consolidation, calls to
+PyMem_RawMalloc() and PyMem_RawFree() made before runtime initialization
+have been crashing.  This is because they rely on the raw memory allocator
+having been initialized already.  Before the runtime state change the
+default raw allocator was initialized statically.  This has now been fixed
+by falling back to the defaults in PyMem_RawMalloc() and PyMem_RawFree().

Objects/obmalloc.c

@@ -421,6 +421,11 @@ PyMem_RawMalloc(size_t size)
      */
     if (size > (size_t)PY_SSIZE_T_MAX)
         return NULL;
+    // If this is used before the Python runtime is initialized, fall
+    // back to the default behavior.
+    if (_PyMem_Raw.malloc == NULL) {
+        return _PyMem_RawMalloc(NULL, size);
+    }
     return _PyMem_Raw.malloc(_PyMem_Raw.ctx, size);
 }
 
@@ -445,6 +450,12 @@ PyMem_RawRealloc(void *ptr, size_t new_size)
 void
 PyMem_RawFree(void *ptr)
 {
+    // If this is used before the Python runtime is initialized, fall
+    // back to the default behavior.
+    if (_PyMem_Raw.free == NULL) {
+        _PyMem_RawFree(NULL, ptr);
+        return;
+    }
     _PyMem_Raw.free(_PyMem_Raw.ctx, ptr);
 }
 

Programs/_testembed.c

@@ -72,6 +72,7 @@ static int test_repeated_init_and_subinterpreters(void)
     return 0;
 }
 
+
 /*****************************************************
  * Test forcing a particular IO encoding
  *****************************************************/
@@ -125,6 +126,27 @@ static int test_forced_io_encoding(void)
     return 0;
 }
 
+/*********************************************************
+ * Test parts of the C-API that work before initialization
+ *********************************************************/
+
+static int test_pre_initialization_api(void)
+{
+    wchar_t *program = Py_DecodeLocale("spam", NULL);
+    if (program == NULL) {
+        fprintf(stderr, "Fatal error: cannot decode program name\n");
+        return 1;
+    }
+    Py_SetProgramName(program);
+
+    Py_Initialize();
+    Py_Finalize();
+
+    PyMem_RawFree(program);
+    return 0;
+}
+
+
 /* *********************************************************
  * List of test cases and the function that implements it.
  *
@@ -146,6 +168,7 @@ struct TestCase
 static struct TestCase TestCases[] = {
     { "forced_io_encoding", test_forced_io_encoding },
     { "repeated_init_and_subinterpreters", test_repeated_init_and_subinterpreters },
+    { "pre_initialization_api", test_pre_initialization_api },
     { NULL, NULL }
 };