bpo-33608: Make sure locks in the runtime are properly re-created. (gh-12245)

Author: ericsnowcurrently

Include/internal/pycore_pystate.h

@@ -229,6 +229,7 @@ typedef struct pyruntimestate {
 PyAPI_DATA(_PyRuntimeState) _PyRuntime;
 PyAPI_FUNC(_PyInitError) _PyRuntimeState_Init(_PyRuntimeState *);
 PyAPI_FUNC(void) _PyRuntimeState_Fini(_PyRuntimeState *);
+PyAPI_FUNC(void) _PyRuntimeState_ReInitThreads(void);
 
 /* Initialize _PyRuntimeState.
    Return NULL on success, or return an error message on failure. */

Modules/posixmodule.c

@@ -429,6 +429,7 @@ PyOS_AfterFork_Child(void)
     PyEval_ReInitThreads();
     _PyImport_ReInitLock();
     _PySignal_AfterFork();
+    _PyRuntimeState_ReInitThreads();
 
     run_at_forkers(_PyInterpreterState_Get()->after_forkers_child, 0);
 }

Python/ceval.c

@@ -174,10 +174,10 @@ PyEval_InitThreads(void)
     PyThread_init_thread();
     create_gil();
     take_gil(_PyThreadState_GET());
-    // Set it to the ID of the main thread of the main interpreter.
-    _PyRuntime.main_thread = PyThread_get_thread_ident();
-    if (!_PyRuntime.ceval.pending.lock) {
-        _PyRuntime.ceval.pending.lock = PyThread_allocate_lock();
+
+    _PyRuntime.ceval.pending.lock = PyThread_allocate_lock();
+    if (_PyRuntime.ceval.pending.lock == NULL) {
+        return Py_FatalError("Can't initialize threads for pending calls");
     }
 }
 
@@ -246,8 +246,11 @@ PyEval_ReInitThreads(void)
         return;
     recreate_gil();
     take_gil(current_tstate);
-    _PyRuntime.main_thread = PyThread_get_thread_ident();
+
     _PyRuntime.ceval.pending.lock = PyThread_allocate_lock();
+    if (_PyRuntime.ceval.pending.lock == NULL) {
+        Py_FatalError("Can't initialize threads for pending calls");
+    }
 
     /* Destroy all threads except the current one */
     _PyThreadState_DeleteExcept(current_tstate);
@@ -362,35 +365,12 @@ _pop_pending_call(int (**func)(void *), void **arg)
 int
 Py_AddPendingCall(int (*func)(void *), void *arg)
 {
-    PyThread_type_lock lock = _PyRuntime.ceval.pending.lock;
-
-    /* try a few times for the lock.  Since this mechanism is used
-     * for signal handling (on the main thread), there is a (slim)
-     * chance that a signal is delivered on the same thread while we
-     * hold the lock during the Py_MakePendingCalls() function.
-     * This avoids a deadlock in that case.
-     * Note that signals can be delivered on any thread.  In particular,
-     * on Windows, a SIGINT is delivered on a system-created worker
-     * thread.
-     * We also check for lock being NULL, in the unlikely case that
-     * this function is called before any bytecode evaluation takes place.
-     */
-    if (lock != NULL) {
-        int i;
-        for (i = 0; i<100; i++) {
-            if (PyThread_acquire_lock(lock, NOWAIT_LOCK))
-                break;
-        }
-        if (i == 100)
-            return -1;
-    }
-
+    PyThread_acquire_lock(_PyRuntime.ceval.pending.lock, WAIT_LOCK);
     int result = _push_pending_call(func, arg);
+    PyThread_release_lock(_PyRuntime.ceval.pending.lock);
 
     /* signal main loop */
     SIGNAL_PENDING_CALLS();
-    if (lock != NULL)
-        PyThread_release_lock(lock);
     return result;
 }
 
@@ -439,15 +419,6 @@ make_pending_calls(void)
     UNSIGNAL_PENDING_CALLS();
     int res = 0;
 
-    if (!_PyRuntime.ceval.pending.lock) {
-        /* initial allocation of the lock */
-        _PyRuntime.ceval.pending.lock = PyThread_allocate_lock();
-        if (_PyRuntime.ceval.pending.lock == NULL) {
-            res = -1;
-            goto error;
-        }
-    }
-
     /* perform a bounded number of calls, in case of recursion */
     for (int i=0; i<NPENDINGCALLS; i++) {
         int (*func)(void *) = NULL;

Python/pystate.c

@@ -60,7 +60,8 @@ _PyRuntimeState_Init_impl(_PyRuntimeState *runtime)
         return _Py_INIT_ERR("Can't initialize threads for cross-interpreter data registry");
     }
 
-    // runtime->main_thread is set in PyEval_InitThreads().
+    // Set it to the ID of the main thread of the main interpreter.
+    runtime->main_thread = PyThread_get_thread_ident();
 
     return _Py_INIT_OK();
 }
@@ -94,6 +95,32 @@ _PyRuntimeState_Fini(_PyRuntimeState *runtime)
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 }
 
+/* This function is called from PyOS_AfterFork_Child to ensure that
+ * newly created child processes do not share locks with the parent.
+ */
+
+void
+_PyRuntimeState_ReInitThreads(void)
+{
+    // This was initially set in _PyRuntimeState_Init().
+    _PyRuntime.main_thread = PyThread_get_thread_ident();
+
+    _PyRuntime.interpreters.mutex = PyThread_allocate_lock();
+    if (_PyRuntime.interpreters.mutex == NULL) {
+        Py_FatalError("Can't initialize lock for runtime interpreters");
+    }
+
+    _PyRuntime.interpreters.main->id_mutex = PyThread_allocate_lock();
+    if (_PyRuntime.interpreters.main->id_mutex == NULL) {
+        Py_FatalError("Can't initialize ID lock for main interpreter");
+    }
+
+    _PyRuntime.xidregistry.mutex = PyThread_allocate_lock();
+    if (_PyRuntime.xidregistry.mutex == NULL) {
+        Py_FatalError("Can't initialize lock for cross-interpreter data registry");
+    }
+}
+
 #define HEAD_LOCK() PyThread_acquire_lock(_PyRuntime.interpreters.mutex, \
                                           WAIT_LOCK)
 #define HEAD_UNLOCK() PyThread_release_lock(_PyRuntime.interpreters.mutex)