ceval: move eval_breaker to per-thread state

The eval_breaker variable is used as a signal to break out of the
interpreter loop to handle signals, asynchronous exceptions, stop for
GC, or release the GIL to let another thread run.

We will be able to have multiple active threads running the interpreter
loop so it's useful to move eval_breaker to per-thread state so that
notifications can target a specific thread.

The specific signals are combined as bits in eval_breaker to simplify
atomic updates.

Author: colesbury

Include/cpython/pystate.h

@@ -117,6 +117,8 @@ struct _ts {
     /* thread status (attached, detached, gc) */
     int status;
 
+    uintptr_t eval_breaker;
+
     /* Has been initialized to a safe state.
 
        In order to be effective, this must be set to 0 during or right

Include/internal/pycore_ceval.h

@@ -24,12 +24,10 @@ extern void _Py_FinishPendingCalls(PyThreadState *tstate);
 extern void _PyEval_InitRuntimeState(struct _ceval_runtime_state *);
 extern void _PyEval_InitState(struct _ceval_state *, PyThread_type_lock);
 extern void _PyEval_FiniState(struct _ceval_state *ceval);
-PyAPI_FUNC(void) _PyEval_SignalReceived(PyInterpreterState *interp);
 PyAPI_FUNC(int) _PyEval_AddPendingCall(
     PyInterpreterState *interp,
     int (*func)(void *),
     void *arg);
-PyAPI_FUNC(void) _PyEval_SignalAsyncExc(PyInterpreterState *interp);
 #ifdef HAVE_FORK
 extern PyStatus _PyEval_ReInitThreads(PyThreadState *tstate);
 #endif

Include/internal/pycore_ceval_state.h

@@ -83,13 +83,6 @@ struct _pending_calls {
 
 struct _ceval_state {
     int recursion_limit;
-    /* This single variable consolidates all requests to break out of
-       the fast path in the eval loop. */
-    _Py_atomic_int eval_breaker;
-    /* Request for dropping the GIL */
-    _Py_atomic_int gil_drop_request;
-    /* The GC is ready to be executed */
-    _Py_atomic_int gc_scheduled;
     struct _pending_calls pending;
 };
 

Include/internal/pycore_gil.h

@@ -28,6 +28,8 @@ struct _gil_runtime_state {
     /* Last PyThreadState holding / having held the GIL. This helps us
        know whether anyone else was scheduled after we dropped the GIL. */
     _Py_atomic_address last_holder;
+    /* Current PyThreadState holding the GIL. Protected by mutex. */
+    PyThreadState *holder;
     /* Whether the GIL is already taken (-1 if uninitialized). This is
        atomic because it can be read without any lock taken in ceval.c. */
     _Py_atomic_int locked;

Include/internal/pycore_pystate.h

@@ -16,6 +16,16 @@ enum _threadstatus {
     _Py_THREAD_GC = 2
 };
 
+enum {
+    EVAL_PLEASE_STOP = 1U << 0,
+    EVAL_PENDING_SIGNALS = 1U << 1,
+    EVAL_PENDING_CALLS = 1U << 2,
+    EVAL_DROP_GIL = 1U << 3,
+    EVAL_ASYNC_EXC = 1U << 4,
+    EVAL_EXPLICIT_MERGE = 1U << 5,
+    EVAL_GC = 1U << 6
+};
+
 /* Check if the current thread is the main thread.
    Use _Py_IsMainInterpreter() to check if it's the main interpreter. */
 static inline int
@@ -147,6 +157,25 @@ PyAPI_FUNC(void) _PyThreadState_DeleteExcept(
     _PyRuntimeState *runtime,
     PyThreadState *tstate);
 
+static inline void
+_PyThreadState_Signal(PyThreadState *tstate, uintptr_t bit)
+{
+    _Py_atomic_or_uintptr(&tstate->eval_breaker, bit);
+}
+
+static inline void
+_PyThreadState_Unsignal(PyThreadState *tstate, uintptr_t bit)
+{
+    _Py_atomic_and_uintptr(&tstate->eval_breaker, ~bit);
+}
+
+static inline int
+_PyThreadState_IsSignalled(PyThreadState *tstate, uintptr_t bit)
+{
+    uintptr_t b = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker);
+    return (b & bit) != 0;
+}
+
 
 static inline void
 _PyThreadState_UpdateTracingState(PyThreadState *tstate)

Include/internal/pycore_runtime.h

@@ -119,6 +119,7 @@ typedef struct pyruntimestate {
     } xidregistry;
 
     unsigned long main_thread;
+    PyThreadState *main_tstate;
 
     PyWideStringList orig_argv;
 

Modules/gcmodule.c

@@ -2258,10 +2258,9 @@ _Py_ScheduleGC(PyInterpreterState *interp)
     if (gcstate->collecting == 1) {
         return;
     }
-    struct _ceval_state *ceval = &interp->ceval;
-    if (!_Py_atomic_load_relaxed(&ceval->gc_scheduled)) {
-        _Py_atomic_store_relaxed(&ceval->gc_scheduled, 1);
-        _Py_atomic_store_relaxed(&ceval->eval_breaker, 1);
+    PyThreadState *tstate = _PyThreadState_GET();
+    if (!_PyThreadState_IsSignalled(tstate, EVAL_GC)) {
+        _PyThreadState_Signal(tstate, EVAL_GC);
     }
 }
 

Modules/signalmodule.c

@@ -276,8 +276,9 @@ trip_signal(int sig_num)
     /* Signals are always handled by the main interpreter */
     PyInterpreterState *interp = _PyInterpreterState_Main();
 
-    /* Notify ceval.c */
-    _PyEval_SignalReceived(interp);
+    /* Notify main thread */
+    _PyRuntimeState *runtime = &_PyRuntime;
+    _PyThreadState_Signal(runtime->main_tstate, EVAL_PENDING_SIGNALS);
 
     /* And then write to the wakeup fd *after* setting all the globals and
        doing the _PyEval_SignalReceived. We used to write to the wakeup fd
@@ -1764,9 +1765,8 @@ PyErr_CheckSignals(void)
        Python code to ensure signals are handled. Checking for the GC here
        allows long running native code to clean cycles created using the C-API
        even if it doesn't run the evaluation loop */
-    struct _ceval_state *interp_ceval_state = &tstate->interp->ceval;
-    if (_Py_atomic_load_relaxed(&interp_ceval_state->gc_scheduled)) {
-        _Py_atomic_store_relaxed(&interp_ceval_state->gc_scheduled, 0);
+    if (_PyThreadState_IsSignalled(tstate, EVAL_GC)) {
+        _PyThreadState_Unsignal(tstate, EVAL_GC);
         _Py_RunGC(tstate);
     }
 

Python/bytecodes.c

@@ -124,8 +124,8 @@ dummy_func(
         inst(RESUME, (--)) {
             assert(tstate->cframe == &cframe);
             assert(frame == cframe.current_frame);
-            if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) {
-                goto handle_eval_breaker;
+            if (oparg < 2) {
+                CHECK_EVAL_BREAKER();
             }
         }
 

Python/ceval.c

@@ -719,7 +719,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 
 #define CHECK_EVAL_BREAKER() \
     _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); \
-    if (_Py_atomic_load_relaxed_int32(eval_breaker)) { \
+    if (!_Py_atomic_uintptr_is_zero(&tstate->eval_breaker)) { \
         goto handle_eval_breaker; \
     }
 
@@ -1073,7 +1073,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
     // for the big switch below (in combination with the EXTRA_CASES macro).
     uint8_t opcode;        /* Current opcode */
     int oparg;         /* Current opcode argument, if any */
-    _Py_atomic_int * const eval_breaker = &tstate->interp->ceval.eval_breaker;
 #ifdef LLTRACE
     int lltrace = 0;
 #endif