bpo-36818: Add PyInterpreterState.runtime field. (gh-13129)

https://bugs.python.org/issue36818

Author: ericsnowcurrently

Include/cpython/pystate.h

@@ -110,9 +110,9 @@ struct _ts {
      * if the thread holds the last reference to the lock, decref'ing the
      * lock will delete the lock, and that may trigger arbitrary Python code
      * if there's a weakref, with a callback, to the lock.  But by this time
-     * _PyRuntime.gilstate.tstate_current is already NULL, so only the simplest
-     * of C code can be allowed to run (in particular it must not be possible to
-     * release the GIL).
+     * _PyRuntimeState.gilstate.tstate_current is already NULL, so only the
+     * simplest of C code can be allowed to run (in particular it must not be
+     * possible to release the GIL).
      * So instead of holding the lock directly, the tstate holds a weakref to
      * the lock:  that's the value of on_delete_data below.  Decref'ing a
      * weakref is harmless.

Include/internal/pycore_object.h

@@ -19,9 +19,10 @@ PyAPI_FUNC(int) _PyDict_CheckConsistency(PyObject *mp, int check_content);
  * NB: While the object is tracked by the collector, it must be safe to call the
  * ob_traverse method.
  *
- * Internal note: _PyRuntime.gc.generation0->_gc_prev doesn't have any bit flags
- * because it's not object header.  So we don't use _PyGCHead_PREV() and
- * _PyGCHead_SET_PREV() for it to avoid unnecessary bitwise operations.
+ * Internal note: _PyRuntimeState.gc.generation0->_gc_prev doesn't have
+ * any bit flags because it's not object header.  So we don't use
+ * _PyGCHead_PREV() and _PyGCHead_SET_PREV() for it to avoid unnecessary
+ * bitwise operations.
  *
  * The PyObject_GC_Track() function is the public version of this macro.
  */

Include/internal/pycore_pylifecycle.h

@@ -39,13 +39,10 @@ extern PyStatus _PyFaulthandler_Init(int enable);
 extern int _PyTraceMalloc_Init(int enable);
 extern PyObject * _PyBuiltin_Init(void);
 extern PyStatus _PySys_Create(
-    _PyRuntimeState *runtime,
     PyInterpreterState *interp,
     PyObject **sysmod_p);
 extern PyStatus _PySys_SetPreliminaryStderr(PyObject *sysdict);
-extern int _PySys_InitMain(
-    _PyRuntimeState *runtime,
-    PyInterpreterState *interp);
+extern int _PySys_InitMain(PyInterpreterState *interp);
 extern PyStatus _PyImport_Init(PyInterpreterState *interp);
 extern PyStatus _PyExc_Init(void);
 extern PyStatus _PyErr_Init(void);
@@ -86,10 +83,7 @@ extern void _PyHash_Fini(void);
 extern int _PyTraceMalloc_Fini(void);
 extern void _PyWarnings_Fini(PyInterpreterState *interp);
 
-extern void _PyGILState_Init(
-    _PyRuntimeState *runtime,
-    PyInterpreterState *interp,
-    PyThreadState *tstate);
+extern void _PyGILState_Init(PyThreadState *tstate);
 extern void _PyGILState_Fini(_PyRuntimeState *runtime);
 
 PyAPI_FUNC(void) _PyGC_DumpShutdownStats(_PyRuntimeState *runtime);

Include/internal/pycore_pystate.h

@@ -19,6 +19,9 @@ extern "C" {
 #include "pycore_pymem.h"
 #include "pycore_warnings.h"
 
+// forward
+struct pyruntimestate;
+
 
 /* ceval state */
 
@@ -68,6 +71,7 @@ struct _is {
 
     struct _is *next;
     struct _ts *tstate_head;
+    struct pyruntimestate *runtime;
 
     int64_t id;
     int64_t id_refcount;
@@ -296,12 +300,8 @@ PyAPI_FUNC(void) _PyRuntime_Finalize(void);
 
 /* Other */
 
-PyAPI_FUNC(void) _PyThreadState_Init(
-    _PyRuntimeState *runtime,
-    PyThreadState *tstate);
-PyAPI_FUNC(void) _PyThreadState_DeleteExcept(
-    _PyRuntimeState *runtime,
-    PyThreadState *tstate);
+PyAPI_FUNC(void) _PyThreadState_Init(PyThreadState *tstate);
+PyAPI_FUNC(void) _PyThreadState_DeleteExcept(PyThreadState *tstate);
 
 PyAPI_FUNC(PyThreadState *) _PyThreadState_Swap(
     struct _gilstate_runtime_state *gilstate,

Misc/NEWS.d/next/Core and Builtins/2019-05-06-14-46-48.bpo-36818.5UDDLj.rst

@@ -0,0 +1 @@
+Add PyInterpreterState.runtime (and use it).

Modules/_threadmodule.c

@@ -996,7 +996,7 @@ t_bootstrap(void *boot_raw)
 
     tstate = boot->tstate;
     tstate->thread_id = PyThread_get_thread_ident();
-    _PyThreadState_Init(&_PyRuntime, tstate);
+    _PyThreadState_Init(tstate);
     PyEval_AcquireThread(tstate);
     tstate->interp->num_threads++;
     res = PyObject_Call(boot->func, boot->args, boot->keyw);

Python/ceval.c

@@ -217,8 +217,9 @@ _PyEval_FiniThreads(struct _ceval_runtime_state *ceval)
 }
 
 static inline void
-exit_thread_if_finalizing(_PyRuntimeState *runtime, PyThreadState *tstate)
+exit_thread_if_finalizing(PyThreadState *tstate)
 {
+    _PyRuntimeState *runtime = tstate->interp->runtime;
     /* _Py_Finalizing is protected by the GIL */
     if (runtime->finalizing != NULL && !_Py_CURRENTLY_FINALIZING(runtime, tstate)) {
         drop_gil(&runtime->ceval, tstate);
@@ -236,7 +237,7 @@ PyEval_AcquireLock(void)
         Py_FatalError("PyEval_AcquireLock: current thread state is NULL");
     }
     take_gil(ceval, tstate);
-    exit_thread_if_finalizing(runtime, tstate);
+    exit_thread_if_finalizing(tstate);
 }
 
 void
@@ -257,14 +258,15 @@ PyEval_AcquireThread(PyThreadState *tstate)
     if (tstate == NULL) {
         Py_FatalError("PyEval_AcquireThread: NULL new thread state");
     }
+    assert(tstate->interp != NULL);
 
-    _PyRuntimeState *runtime = &_PyRuntime;
+    _PyRuntimeState *runtime = tstate->interp->runtime;
     struct _ceval_runtime_state *ceval = &runtime->ceval;
 
     /* Check someone has called PyEval_InitThreads() to create the lock */
     assert(gil_created(&ceval->gil));
     take_gil(ceval, tstate);
-    exit_thread_if_finalizing(runtime, tstate);
+    exit_thread_if_finalizing(tstate);
     if (_PyThreadState_Swap(&runtime->gilstate, tstate) != NULL) {
         Py_FatalError("PyEval_AcquireThread: non-NULL old thread state");
     }
@@ -276,8 +278,9 @@ PyEval_ReleaseThread(PyThreadState *tstate)
     if (tstate == NULL) {
         Py_FatalError("PyEval_ReleaseThread: NULL thread state");
     }
+    assert(tstate->interp != NULL);
 
-    _PyRuntimeState *runtime = &_PyRuntime;
+    _PyRuntimeState *runtime = tstate->interp->runtime;
     PyThreadState *new_tstate = _PyThreadState_Swap(&runtime->gilstate, NULL);
     if (new_tstate != tstate) {
         Py_FatalError("PyEval_ReleaseThread: wrong thread state");
@@ -308,7 +311,7 @@ _PyEval_ReInitThreads(_PyRuntimeState *runtime)
     }
 
     /* Destroy all threads except the current one */
-    _PyThreadState_DeleteExcept(runtime, current_tstate);
+    _PyThreadState_DeleteExcept(current_tstate);
 }
 
 /* This function is used to signal that async exceptions are waiting to be
@@ -337,17 +340,18 @@ PyEval_SaveThread(void)
 void
 PyEval_RestoreThread(PyThreadState *tstate)
 {
-    _PyRuntimeState *runtime = &_PyRuntime;
-    struct _ceval_runtime_state *ceval = &runtime->ceval;
-
     if (tstate == NULL) {
         Py_FatalError("PyEval_RestoreThread: NULL tstate");
     }
+    assert(tstate->interp != NULL);
+
+    _PyRuntimeState *runtime = tstate->interp->runtime;
+    struct _ceval_runtime_state *ceval = &runtime->ceval;
     assert(gil_created(&ceval->gil));
 
     int err = errno;
     take_gil(ceval, tstate);
-    exit_thread_if_finalizing(runtime, tstate);
+    exit_thread_if_finalizing(tstate);
     errno = err;
 
     _PyThreadState_Swap(&runtime->gilstate, tstate);
@@ -1141,7 +1145,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
                 take_gil(ceval, tstate);
 
                 /* Check if we should make a quick exit. */
-                exit_thread_if_finalizing(runtime, tstate);
+                exit_thread_if_finalizing(tstate);
 
                 if (_PyThreadState_Swap(&runtime->gilstate, tstate) != NULL) {
                     Py_FatalError("ceval: orphan tstate");

Python/import.c

@@ -541,7 +541,8 @@ PyImport_Cleanup(void)
     _PyGC_CollectNoFail();
     /* Dump GC stats before it's too late, since it uses the warnings
        machinery. */
-    _PyGC_DumpShutdownStats(&_PyRuntime);
+    _PyRuntimeState *runtime = interp->runtime;
+    _PyGC_DumpShutdownStats(runtime);
 
     /* Now, if there are any modules left alive, clear their globals to
        minimize potential leaks.  All C extension modules actually end

Python/pylifecycle.c

@@ -545,7 +545,7 @@ pycore_create_interpreter(_PyRuntimeState *runtime,
     _PyEval_FiniThreads(&runtime->ceval);
 
     /* Auto-thread-state API */
-    _PyGILState_Init(runtime, interp, tstate);
+    _PyGILState_Init(tstate);
 
     /* Create the GIL */
     PyEval_InitThreads();
@@ -683,7 +683,7 @@ pyinit_config(_PyRuntimeState *runtime,
     }
 
     PyObject *sysmod;
-    status = _PySys_Create(runtime, interp, &sysmod);
+    status = _PySys_Create(interp, &sysmod);
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
@@ -892,8 +892,9 @@ _Py_ReconfigureMainInterpreter(PyInterpreterState *interp)
  * non-zero return code.
  */
 static PyStatus
-pyinit_main(_PyRuntimeState *runtime, PyInterpreterState *interp)
+pyinit_main(PyInterpreterState *interp)
 {
+    _PyRuntimeState *runtime = interp->runtime;
     if (!runtime->core_initialized) {
         return _PyStatus_ERR("runtime core not initialized");
     }
@@ -919,7 +920,7 @@ pyinit_main(_PyRuntimeState *runtime, PyInterpreterState *interp)
         return _PyStatus_ERR("can't initialize time");
     }
 
-    if (_PySys_InitMain(runtime, interp) < 0) {
+    if (_PySys_InitMain(interp) < 0) {
         return _PyStatus_ERR("can't finish initializing sys");
     }
 
@@ -999,7 +1000,7 @@ _Py_InitializeMain(void)
     _PyRuntimeState *runtime = &_PyRuntime;
     PyInterpreterState *interp = _PyRuntimeState_GetThreadState(runtime)->interp;
 
-    return pyinit_main(runtime, interp);
+    return pyinit_main(interp);
 }
 
 
@@ -1026,7 +1027,7 @@ Py_InitializeFromConfig(const PyConfig *config)
     config = &interp->config;
 
     if (config->_init_main) {
-        status = pyinit_main(runtime, interp);
+        status = pyinit_main(interp);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
@@ -1453,7 +1454,7 @@ new_interpreter(PyThreadState **tstate_p)
         }
         Py_INCREF(interp->sysdict);
         PyDict_SetItemString(interp->sysdict, "modules", modules);
-        if (_PySys_InitMain(runtime, interp) < 0) {
+        if (_PySys_InitMain(interp) < 0) {
             return _PyStatus_ERR("can't finish initializing sys");
         }
     }