bpo-39984: Move pending calls to PyInterpreterState (GH-19066)

If Py_AddPendingCall() is called in a subinterpreter, the function is
now scheduled to be called from the subinterpreter, rather than being
called from the main interpreter.

Each subinterpreter now has its own list of scheduled calls.

* Move pending and eval_breaker fields from _PyRuntimeState.ceval
  to PyInterpreterState.ceval.
* new_interpreter() now calls _PyEval_InitThreads() to create
  pending calls lock.
* Fix Py_AddPendingCall() for subinterpreters. It now calls
  _PyThreadState_GET() which works in a subinterpreter if the
  caller holds the GIL, and only falls back on
  PyGILState_GetThisThreadState() if _PyThreadState_GET()
  returns NULL.

Author: vstinner

Doc/c-api/init.rst

@@ -1389,6 +1389,10 @@ pointer and a void pointer argument.
    This function doesn't need a current thread state to run, and it doesn't
    need the global interpreter lock.
 
+   To call this function in a subinterpreter, the caller must hold the GIL.
+   Otherwise, the function *func* can be scheduled to be called from the wrong
+   interpreter.
+
    .. warning::
       This is a low-level function, only useful for very special cases.
       There is no guarantee that *func* will be called as quick as
@@ -1397,6 +1401,12 @@ pointer and a void pointer argument.
       function is generally **not** suitable for calling Python code from
       arbitrary C threads.  Instead, use the :ref:`PyGILState API<gilstate>`.
 
+   .. versionchanged:: 3.9
+      If this function is called in a subinterpreter, the function *func* is
+      now scheduled to be called from the subinterpreter, rather than being
+      called from the main interpreter. Each subinterpreter now has its own
+      list of scheduled calls.
+
    .. versionadded:: 3.1
 
 .. _profiling:

Doc/whatsnew/3.9.rst

@@ -514,6 +514,12 @@ Build and C API Changes
 
   Extension modules without module state (``m_size <= 0``) are not affected.
 
+* If :c:func:`Py_AddPendingCall` is called in a subinterpreter, the function is
+  now scheduled to be called from the subinterpreter, rather than being called
+  from the main interpreter. Each subinterpreter now has its own list of
+  scheduled calls.
+  (Contributed by Victor Stinner in :issue:`39984`.)
+
 
 Deprecated
 ==========

Include/internal/pycore_pystate.h

@@ -35,12 +35,8 @@ struct _pending_calls {
 
 struct _ceval_runtime_state {
     int recursion_limit;
-    /* This single variable consolidates all requests to break out of
-       the fast path in the eval loop. */
-    _Py_atomic_int eval_breaker;
     /* Request for dropping the GIL */
     _Py_atomic_int gil_drop_request;
-    struct _pending_calls pending;
     /* Request for checking signals. */
     _Py_atomic_int signals_pending;
     struct _gil_runtime_state gil;
@@ -53,6 +49,10 @@ struct _ceval_state {
        c_tracefunc.  This speeds up the if statement in
        _PyEval_EvalFrameDefault() after fast_next_opcode. */
     int tracing_possible;
+    /* This single variable consolidates all requests to break out of
+       the fast path in the eval loop. */
+    _Py_atomic_int eval_breaker;
+    struct _pending_calls pending;
 };
 
 /* interpreter state */

Misc/NEWS.d/next/Core and Builtins/2020-03-19-00-45-37.bpo-39984.u-bHIq.rst

@@ -0,0 +1,4 @@
+If :c:func:`Py_AddPendingCall` is called in a subinterpreter, the function is
+now scheduled to be called from the subinterpreter, rather than being called
+from the main interpreter. Each subinterpreter now has its own list of
+scheduled calls.

Modules/signalmodule.c

@@ -304,7 +304,7 @@ trip_signal(int sig_num)
                 if (wakeup.warn_on_full_buffer ||
                     last_error != WSAEWOULDBLOCK)
                 {
-                    /* Py_AddPendingCall() isn't signal-safe, but we
+                    /* _PyEval_AddPendingCall() isn't signal-safe, but we
                        still use it for this exceptional case. */
                     _PyEval_AddPendingCall(tstate,
                                            report_wakeup_send_error,
@@ -323,7 +323,7 @@ trip_signal(int sig_num)
                 if (wakeup.warn_on_full_buffer ||
                     (errno != EWOULDBLOCK && errno != EAGAIN))
                 {
-                    /* Py_AddPendingCall() isn't signal-safe, but we
+                    /* _PyEval_AddPendingCall() isn't signal-safe, but we
                        still use it for this exceptional case. */
                     _PyEval_AddPendingCall(tstate,
                                            report_wakeup_write_error,