Add safe memory reclamation scheme based on FreeBSD's GUS

The scheme will be used to allow safe reads from dicts and lists that
don't acquire the collection's lock.

Author: colesbury

Include/internal/pycore_interp.h

@@ -68,6 +68,8 @@ typedef struct PyThreadStateImpl {
     PyThreadState tstate;
 
     struct brc_state brc;
+
+    struct qsbr *qsbr;
 } PyThreadStateImpl;
 
 

Include/internal/pycore_qsbr.h

@@ -0,0 +1,65 @@
+#ifndef Py_INTERNAL_QSBR_H
+#define Py_INTERNAL_QSBR_H
+
+#include <stdbool.h>
+#include "pyatomic.h"
+#include "pycore_llist.h"
+#include "pycore_initconfig.h"
+#include "pycore_pystate.h"
+#include "pycore_runtime.h"
+
+/* Per-thread state */
+struct qsbr {
+    uint64_t            t_seq;
+    struct qsbr_shared  *t_shared;
+    struct qsbr         *t_next;
+    PyThreadState       *tstate;
+};
+
+struct qsbr_pad {
+    struct qsbr qsbr;
+    char __padding[64 - sizeof(struct qsbr)];
+};
+
+static inline uint64_t
+_Py_qsbr_shared_current(struct qsbr_shared *shared)
+{
+    return _Py_atomic_load_uint64(&shared->s_wr);
+}
+
+static inline void
+_Py_qsbr_quiescent_state(PyThreadState *ts)
+{
+    struct qsbr *qsbr = ((struct PyThreadStateImpl *)ts)->qsbr;
+    uint64_t seq = _Py_qsbr_shared_current(qsbr->t_shared); // need acquire
+    _Py_atomic_store_uint64_relaxed(&qsbr->t_seq, seq); // probably release
+}
+
+PyStatus
+_Py_qsbr_init(struct qsbr_shared *shared);
+
+uint64_t
+_Py_qsbr_advance(struct qsbr_shared *shared);
+
+bool
+_Py_qsbr_poll(struct qsbr *qsbr, uint64_t goal);
+
+void
+_Py_qsbr_online(struct qsbr *qsbr);
+
+void
+_Py_qsbr_offline(struct qsbr *qsbr);
+
+struct qsbr *
+_Py_qsbr_recycle(struct qsbr_shared *shared, PyThreadState *tsate);
+
+struct qsbr *
+_Py_qsbr_register(struct qsbr_shared *shared, PyThreadState *tsate, struct qsbr *qsbr);
+
+void
+_Py_qsbr_unregister(struct qsbr *qsbr);
+
+void
+_Py_qsbr_after_fork(struct qsbr_shared *shared, struct qsbr *qsbr);
+
+#endif /* !Py_INTERNAL_QSBR_H */

Include/internal/pycore_runtime.h

@@ -56,6 +56,9 @@ typedef struct _Py_AuditHookEntry {
     void *userData;
 } _Py_AuditHookEntry;
 
+/* See pycore_qsbr.h for full definition */
+struct qsbr;
+
 /* Full Python runtime state */
 
 /* _PyRuntimeState holds the global state for the CPython runtime.
@@ -118,6 +121,17 @@ typedef struct pyruntimestate {
         struct _xidregitem *head;
     } xidregistry;
 
+    struct qsbr_shared {
+        /* always odd, incremented by two */
+        uint64_t s_wr;
+
+        /* Minimum observed read sequence. */
+        uint64_t s_rd_seq;
+
+        struct qsbr *head;
+        uintptr_t n_free;
+    } qsbr_shared;
+
     unsigned long main_thread;
     PyThreadState *main_tstate;
 

Makefile.pre.in

@@ -415,6 +415,7 @@ PYTHON_OBJS=	\
 		Python/pyrefcnt.o \
 		Python/pythonrun.o \
 		Python/pytime.o \
+		Python/qsbr.o \
 		Python/bootstrap_hash.o \
 		Python/specialize.o \
 		Python/structmember.o \

PCbuild/_freeze_module.vcxproj

@@ -232,6 +232,7 @@
     <ClCompile Include="..\Python\pythonrun.c" />
     <ClCompile Include="..\Python\Python-tokenize.c" />
     <ClCompile Include="..\Python\pytime.c" />
+    <ClCompile Include="..\Python\qsbr.c" />
     <ClCompile Include="..\Python\specialize.c" />
     <ClCompile Include="..\Python\structmember.c" />
     <ClCompile Include="..\Python\suggestions.c" />

PCbuild/_freeze_module.vcxproj.filters

@@ -340,6 +340,9 @@
     <ClCompile Include="..\Python\pytime.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\Python\qsbr.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\Objects\rangeobject.c">
       <Filter>Source Files</Filter>
     </ClCompile>

PCbuild/pythoncore.vcxproj

@@ -253,6 +253,7 @@
     <ClInclude Include="..\Include\internal\pycore_pymem_init.h" />
     <ClInclude Include="..\Include\internal\pycore_pystate.h" />
     <ClInclude Include="..\Include\internal\pycore_pythread.h" />
+    <ClInclude Include="..\Include\internal\pycore_qsbr.h" />
     <ClInclude Include="..\Include\internal\pycore_range.h" />
     <ClInclude Include="..\Include\internal\pycore_refcnt.h" />
     <ClInclude Include="..\Include\internal\pycore_runtime.h" />
@@ -556,6 +557,7 @@
     <ClCompile Include="..\Python\Python-ast.c" />
     <ClCompile Include="..\Python\Python-tokenize.c" />
     <ClCompile Include="..\Python\pythonrun.c" />
+    <ClCompile Include="..\Python\qsbr.c" />
     <ClCompile Include="..\Python\specialize.c" />
     <ClCompile Include="..\Python\suggestions.c" />
     <ClCompile Include="..\Python\structmember.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -663,6 +663,9 @@
     <ClInclude Include="..\Include\internal\pycore_pythread.h">
       <Filter>Include\internal</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\internal\pycore_qsbr.h">
+      <Filter>Include</Filter>
+    </ClInclude>
     <ClInclude Include="..\Include\internal\pycore_range.h">
       <Filter>Include\internal</Filter>
     </ClInclude>
@@ -1253,6 +1256,9 @@
     <ClCompile Include="..\Python\pythonrun.c">
       <Filter>Python</Filter>
     </ClCompile>
+    <ClCompile Include="..\Python\qsbr.c">
+      <Filter>Python</Filter>
+    </ClCompile>
     <ClCompile Include="..\Python\specialize.c">
       <Filter>Python</Filter>
     </ClCompile>

Python/pylifecycle.c

@@ -21,6 +21,7 @@
 #include "pycore_pylifecycle.h"   // _PyErr_Print()
 #include "pycore_pymem.h"         // _PyObject_DebugMallocStats()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
+#include "pycore_qsbr.h"          // _Py_qsbr_init()
 #include "pycore_runtime.h"       // _Py_ID()
 #include "pycore_runtime_init.h"  // _PyRuntimeState_INIT
 #include "pycore_sliceobject.h"   // _PySlice_Fini()
@@ -619,6 +620,11 @@ pycore_init_runtime(_PyRuntimeState *runtime,
     if (_PyStatus_EXCEPTION(status)) {
         return status;
     }
+
+    status = _Py_qsbr_init(&runtime->qsbr_shared);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
     return _PyStatus_OK();
 }
 

Python/pystate.c

@@ -13,6 +13,7 @@
 #include "pycore_pylifecycle.h"
 #include "pycore_pymem.h"         // _PyMem_SetDefaultAllocator()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
+#include "pycore_qsbr.h"
 #include "pycore_runtime_init.h"  // _PyRuntimeState_INIT
 #include "pycore_sysmodule.h"
 #include "pycore_refcnt.h"
@@ -164,12 +165,13 @@ _PyThreadState_Attach(PyThreadState *tstate)
             &tstate->status,
             _Py_THREAD_DETACHED,
             _Py_THREAD_ATTACHED)) {
+        // online for QSBR too
+        _Py_qsbr_online(((PyThreadStateImpl *)tstate)->qsbr);
 
         // resume previous critical section
         if (tstate->critical_section != 0) {
             _Py_critical_section_resume(tstate);
         }
-
         return 1;
     }
     return 0;
@@ -178,6 +180,8 @@ _PyThreadState_Attach(PyThreadState *tstate)
 static void
 _PyThreadState_Detach(PyThreadState *tstate)
 {
+    _Py_qsbr_offline(((PyThreadStateImpl *)tstate)->qsbr);
+
     if (tstate->critical_section != 0) {
         _Py_critical_section_end_all(tstate);
     }
@@ -737,7 +741,8 @@ free_threadstate(PyThreadState *tstate)
 static void
 init_threadstate(PyThreadState *tstate,
                  PyInterpreterState *interp, uint64_t id,
-                 PyThreadState *next)
+                 PyThreadState *next,
+                 struct qsbr *empty_qsbr)
 {
     if (tstate->_initialized) {
         Py_FatalError("thread state already initialized");
@@ -763,6 +768,18 @@ init_threadstate(PyThreadState *tstate,
     tstate->native_thread_id = PyThread_get_thread_native_id();
 #endif
 
+    // First try to recycle an existing qsbr structure
+    PyThreadStateImpl *tstate_impl = (PyThreadStateImpl *)tstate;
+    struct qsbr *recycled = _Py_qsbr_recycle(&_PyRuntime.qsbr_shared, tstate);
+    if (recycled) {
+        tstate_impl->qsbr = recycled;
+    }
+    else {
+        // If no recycled struct, use the newly allocated empty qsbr struct
+        tstate_impl->qsbr = empty_qsbr;
+        _Py_qsbr_register(&_PyRuntime.qsbr_shared, tstate, empty_qsbr);
+    }
+
     tstate->py_recursion_limit = interp->ceval.recursion_limit,
     tstate->py_recursion_remaining = interp->ceval.recursion_limit,
     tstate->c_recursion_remaining = C_RECURSION_LIMIT;
@@ -791,6 +808,12 @@ new_threadstate(PyInterpreterState *interp)
     if (new_tstate == NULL) {
         return NULL;
     }
+    struct qsbr *qsbr = PyMem_RawCalloc(1, sizeof(struct qsbr_pad));
+    if (qsbr == NULL) {
+        PyMem_RawFree(new_tstate);
+        return NULL;
+    }
+
     /* We serialize concurrent creation to protect global state. */
     HEAD_LOCK(runtime);
 
@@ -818,13 +841,17 @@ new_threadstate(PyInterpreterState *interp)
     }
     interp->threads.head = tstate;
 
-    init_threadstate(tstate, interp, id, old_head);
+    init_threadstate(tstate, interp, id, old_head, qsbr);
 
     HEAD_UNLOCK(runtime);
     if (!used_newtstate) {
         // Must be called with lock unlocked to avoid re-entrancy deadlock.
         PyMem_RawFree(new_tstate);
     }
+    if (qsbr->tstate == NULL) {
+        // If the qsbr structure wasn't used, free it here after the unlock.
+        PyMem_RawFree(qsbr);
+    }
     return tstate;
 }
 
@@ -1062,6 +1089,13 @@ tstate_delete_common(PyThreadState *tstate,
         PyThread_tss_set(&gilstate->autoTSSkey, NULL);
     }
 
+    PyThreadStateImpl *tstate_impl = (PyThreadStateImpl *)tstate;
+    if (is_current) {
+        _Py_qsbr_offline(tstate_impl->qsbr);
+    }
+    _Py_qsbr_unregister(tstate_impl->qsbr);
+    tstate_impl->qsbr = NULL;
+
     _PyRuntimeState *runtime = interp->runtime;
     HEAD_LOCK(runtime);
     if (tstate->prev) {