bpo-33608: Factor out a private, per-interpreter _Py_AddPendingCall(). (gh-13714)

Author: ericsnowcurrently

Include/internal/pycore_ceval.h

@@ -12,19 +12,22 @@ extern "C" {
 #include "pycore_pystate.h"
 #include "pythread.h"
 
-PyAPI_FUNC(void) _Py_FinishPendingCalls(_PyRuntimeState *runtime);
 PyAPI_FUNC(void) _PyEval_Initialize(struct _ceval_runtime_state *);
 PyAPI_FUNC(void) _PyEval_FiniThreads(
-    struct _ceval_runtime_state *ceval);
+    struct _ceval_runtime_state *);
 PyAPI_FUNC(void) _PyEval_SignalReceived(
-    struct _ceval_runtime_state *ceval);
+    struct _ceval_runtime_state *);
 PyAPI_FUNC(int) _PyEval_AddPendingCall(
     PyThreadState *tstate,
-    struct _ceval_runtime_state *ceval,
+    struct _ceval_runtime_state *,
+    struct _ceval_interpreter_state *,
+    unsigned long thread_id,
     int (*func)(void *),
     void *arg);
+PyAPI_FUNC(void) _PyEval_FinishPendingCalls(PyInterpreterState *);
 PyAPI_FUNC(void) _PyEval_SignalAsyncExc(
-    struct _ceval_runtime_state *ceval);
+    struct _ceval_runtime_state *,
+    struct _ceval_interpreter_state *);
 PyAPI_FUNC(void) _PyEval_ReInitThreads(
     _PyRuntimeState *runtime);
 

Include/internal/pycore_pystate.h

@@ -25,7 +25,7 @@ struct pyruntimestate;
 
 /* ceval state */
 
-struct _pending_calls {
+struct _ceval_pending_calls {
     int finishing;
     PyThread_type_lock lock;
     /* Request for running pending calls. */
@@ -36,6 +36,7 @@ struct _pending_calls {
     int async_exc;
 #define NPENDINGCALLS 32
     struct {
+        unsigned long thread_id;
         int (*func)(void *);
         void *arg;
     } calls[NPENDINGCALLS];
@@ -53,15 +54,21 @@ struct _ceval_runtime_state {
     int tracing_possible;
     /* This single variable consolidates all requests to break out of
        the fast path in the eval loop. */
+    // XXX This can move to _ceval_interpreter_state once all parts
+    // from COMPUTE_EVAL_BREAKER have moved under PyInterpreterState.
     _Py_atomic_int eval_breaker;
     /* Request for dropping the GIL */
     _Py_atomic_int gil_drop_request;
-    struct _pending_calls pending;
     /* Request for checking signals. */
     _Py_atomic_int signals_pending;
     struct _gil_runtime_state gil;
 };
 
+struct _ceval_interpreter_state {
+    struct _ceval_pending_calls pending;
+};
+
+
 /* interpreter state */
 
 typedef PyObject* (*_PyFrameEvalFunction)(struct _frame *, int);
@@ -136,6 +143,7 @@ struct _is {
 
     uint64_t tstate_next_unique_id;
 
+    struct _ceval_interpreter_state ceval;
     struct _warnings_runtime_state warnings;
 
     PyObject *audit_hooks;

Lib/test/test_capi.py

@@ -431,7 +431,7 @@ def pendingcalls_wait(self, l, n, context = None):
     def test_pendingcalls_threaded(self):
 
         #do every callback on a separate thread
-        n = 32 #total callbacks
+        n = 32 #total callbacks (see NPENDINGCALLS in pycore_ceval.h)
         threads = []
         class foo(object):pass
         context = foo()

Misc/NEWS.d/next/Core and Builtins/2018-09-15-12-13-46.bpo-33608.avmvVP.rst

@@ -0,0 +1,5 @@
+We added a new internal _Py_AddPendingCall() that operates relative to the
+provided interpreter.  This allows us to use the existing implementation to
+ask another interpreter to do work that cannot be done in the current
+interpreter, like decref an object the other interpreter owns.  The existing
+Py_AddPendingCall() only operates relative to the main interpreter.

Modules/_testcapimodule.c

@@ -2677,6 +2677,7 @@ pending_threadfunc(PyObject *self, PyObject *arg)
     Py_INCREF(callable);
 
     Py_BEGIN_ALLOW_THREADS
+    /* XXX Use the internal _Py_AddPendingCall(). */
     r = Py_AddPendingCall(&_pending_callback, callable);
     Py_END_ALLOW_THREADS
 

Modules/signalmodule.c

@@ -21,6 +21,7 @@
 #include <process.h>
 #endif
 #endif
+#include "internal/pycore_pystate.h"
 
 #ifdef HAVE_SIGNAL_H
 #include <signal.h>
@@ -259,6 +260,7 @@ trip_signal(int sig_num)
     /* Notify ceval.c */
     _PyRuntimeState *runtime = &_PyRuntime;
     PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
+    PyInterpreterState *interp = runtime->interpreters.main;
     _PyEval_SignalReceived(&runtime->ceval);
 
     /* And then write to the wakeup fd *after* setting all the globals and
@@ -299,7 +301,10 @@ trip_signal(int sig_num)
                 {
                     /* Py_AddPendingCall() isn't signal-safe, but we
                        still use it for this exceptional case. */
-                    _PyEval_AddPendingCall(tstate, &runtime->ceval,
+                    _PyEval_AddPendingCall(tstate,
+                                           &runtime->ceval,
+                                           &interp->ceval,
+                                           runtime->main_thread,
                                            report_wakeup_send_error,
                                            (void *)(intptr_t) last_error);
                 }
@@ -318,7 +323,10 @@ trip_signal(int sig_num)
                 {
                     /* Py_AddPendingCall() isn't signal-safe, but we
                        still use it for this exceptional case. */
-                    _PyEval_AddPendingCall(tstate, &runtime->ceval,
+                    _PyEval_AddPendingCall(tstate,
+                                           &runtime->ceval,
+                                           &interp->ceval,
+                                           runtime->main_thread,
                                            report_wakeup_write_error,
                                            (void *)(intptr_t)errno);
                 }