mro: thread-safe MRO cache

Author: colesbury

Include/cpython/object.h

@@ -136,6 +136,14 @@ typedef struct {
  * backwards-compatibility */
 typedef Py_ssize_t printfunc;
 
+struct _Py_mro_cache_entry;
+
+typedef struct {
+    struct _Py_mro_cache_entry *buckets;
+    uint32_t mask;
+} _Py_mro_cache;
+
+
 // If this structure is modified, Doc/includes/typestruct.h should be updated
 // as well.
 struct _typeobject {
@@ -221,6 +229,9 @@ struct _typeobject {
     destructor tp_finalize;
     vectorcallfunc tp_vectorcall;
 
+    /* Added in version 3.13 */
+    _Py_mro_cache tp_mro_cache;
+
     /* bitset of which type-watchers care about this type */
     char tp_watched;
 };

Include/internal/pycore_interp.h

@@ -23,6 +23,7 @@ extern "C" {
 #include "pycore_gc.h"            // struct _gc_runtime_state
 #include "pycore_list.h"          // struct _Py_list_state
 #include "pycore_llist.h"         // struct llist_node
+#include "pycore_mrocache.h"      // struct _mro_cache_state
 #include "pycore_global_objects.h"  // struct _Py_interp_static_objects
 #include "pycore_pymem.h"         // struct _mem_work
 #include "pycore_tuple.h"         // struct _Py_tuple_state
@@ -78,8 +79,6 @@ typedef struct PyThreadStateImpl {
     struct brc_state brc;
 
     struct qsbr *qsbr;
-
-    struct type_cache type_cache;
 } PyThreadStateImpl;
 
 
@@ -127,6 +126,7 @@ struct _is {
     struct _ceval_state ceval;
     struct _gc_runtime_state gc;
     struct _mem_state mem;
+    struct _mro_cache_state mro_cache;
 
     // sys.modules dictionary
     PyObject *modules;
@@ -211,6 +211,8 @@ struct _is {
     struct callable_cache callable_cache;
     PyCodeObject *interpreter_trampoline;
 
+    struct _Py_queue_head mro_buckets_to_free;
+
     struct _Py_interp_cached_objects cached_objects;
     struct _Py_interp_static_objects static_objects;
 

Include/internal/pycore_mrocache.h

@@ -0,0 +1,110 @@
+#ifndef Py_INTERNAL_TYPECACHE_H
+#define Py_INTERNAL_TYPECACHE_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+// TODO(sgross): MRO cache or type cache?
+
+typedef struct _Py_mro_cache_entry {
+    PyObject *name;     /* name (interned unicode; immortal) */
+    uintptr_t value;    /* resolved function (owned ref), or 0=not cached 1=not present */
+} _Py_mro_cache_entry;
+
+typedef struct _Py_mro_cache_buckets {
+    struct _Py_queue_node node;
+    union {
+        Py_ssize_t refcount;
+        Py_ssize_t capacity;
+    } u;
+    uint32_t available; /* number of unused buckets */
+    uint32_t used;      /* number of used buckets */
+    _Py_mro_cache_entry array[];
+} _Py_mro_cache_buckets;
+
+/* Per-interpreter state */
+struct _mro_cache_state {
+    _Py_mro_cache_buckets *empty_buckets;
+    Py_ssize_t empty_buckets_capacity;
+};
+
+typedef struct _Py_mro_cache_result {
+    int hit;
+    PyObject *value;
+} _Py_mro_cache_result;
+
+extern PyStatus _Py_mro_cache_init(PyInterpreterState *interp);
+extern void _Py_mro_cache_fini(PyInterpreterState *interp);
+extern void _Py_mro_cache_init_type(PyTypeObject *type);
+extern void _Py_mro_cache_fini_type(PyTypeObject *type);
+extern int _Py_mro_cache_visit(_Py_mro_cache *cache, visitproc visit, void *arg);
+
+extern void _Py_mro_cache_erase(_Py_mro_cache *cache);
+extern void _Py_mro_cache_insert(_Py_mro_cache *cache, PyObject *name, PyObject *value);
+extern void _Py_mro_process_freed_buckets(PyInterpreterState *interp);
+
+extern PyObject *_Py_mro_cache_as_dict(_Py_mro_cache *cache);
+
+static inline _Py_mro_cache_result
+_Py_mro_cache_make_result(uintptr_t *ptr)
+{
+    uintptr_t value = _Py_atomic_load_uintptr_relaxed(ptr);
+    return (_Py_mro_cache_result) {
+        .hit = value != 0,
+        .value = (PyObject *)(value & ~1),
+    };
+}
+
+static inline struct _Py_mro_cache_result
+_Py_mro_cache_lookup(_Py_mro_cache *cache, PyObject *name)
+{
+    Py_hash_t hash = ((PyASCIIObject *)name)->hash;
+    uint32_t mask = _Py_atomic_load_uint32(&cache->mask);
+    _Py_mro_cache_entry *first = _Py_atomic_load_ptr_relaxed(&cache->buckets);
+
+    Py_ssize_t offset = hash & mask;
+    _Py_mro_cache_entry *bucket = (_Py_mro_cache_entry *)((char *)first + offset);
+
+    PyObject *entry_name = _Py_atomic_load_ptr_relaxed(&bucket->name);
+    if (_PY_LIKELY(entry_name == name)) {
+        return _Py_mro_cache_make_result(&bucket->value);
+    }
+
+    /* First loop */
+    while (1) {
+        if (entry_name == NULL) {
+            return (_Py_mro_cache_result){0, NULL};
+        }
+        if (bucket == first) {
+            break;
+        }
+        bucket--;
+        entry_name = _Py_atomic_load_ptr_relaxed(&bucket->name);
+        if (entry_name == name) {
+            return _Py_mro_cache_make_result(&bucket->value);
+        }
+    }
+
+    /* Second loop. Start at the last bucket. */
+    bucket = (_Py_mro_cache_entry *)((char *)first + mask);
+    while (1) {
+        entry_name = _Py_atomic_load_ptr_relaxed(&bucket->name);
+        if (entry_name == name) {
+            return _Py_mro_cache_make_result(&bucket->value);
+        }
+        if (entry_name == NULL || bucket == first) {
+            return (_Py_mro_cache_result){0, NULL};
+        }
+        bucket--;
+    }
+}
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif   /* !Py_INTERNAL_TYPECACHE_H */

Include/internal/pycore_pymem.h

@@ -101,6 +101,7 @@ PyAPI_FUNC(int) _PyMem_SetupAllocators(PyMemAllocatorName allocator);
 
 /* Free the pointer after all threads are quiescent. */
 extern void _PyMem_FreeQsbr(void *ptr);
+extern void _PyQsbr_Free(void *ptr, freefunc func);
 extern void _PyMem_QsbrPoll(PyThreadState *tstate);
 extern void _PyMem_AbandonQsbr(PyThreadState *tstate);
 extern void _PyMem_QsbrFini(PyInterpreterState *interp);

Include/internal/pycore_pyqueue.h

@@ -13,6 +13,8 @@ extern "C" {
 // struct _Py_queue_head which contains pointers to the first and
 // last node in the queue.
 
+#define _Py_QUEUE_INIT(name) { { NULL }, &name.first }
+
 static inline void
 _Py_queue_init(struct _Py_queue_head *head)
 {

Include/internal/pycore_typeobject.h

@@ -26,21 +26,6 @@ extern void _PyTypes_Fini(PyInterpreterState *);
 
 typedef struct wrapperbase pytype_slotdef;
 
-
-// Type attribute lookup cache: speed up attribute and method lookups,
-// see _PyType_Lookup().
-struct type_cache_entry {
-    unsigned int version;  // initialized from type->tp_version_tag
-    PyObject *name;        // reference to exactly a str or None
-    PyObject *value;       // borrowed reference or NULL
-};
-
-#define MCACHE_SIZE_EXP 12
-
-struct type_cache {
-    struct type_cache_entry hashtable[1 << MCACHE_SIZE_EXP];
-};
-
 /* For now we hard-code this to a value for which we are confident
    all the static builtin types will fit (for all builds). */
 #define _Py_MAX_STATIC_BUILTIN_TYPES 200
@@ -63,9 +48,6 @@ _PyStaticType_GET_WEAKREFS_LISTPTR(static_builtin_state *state)
 }
 
 struct types_state {
-#ifndef Py_NOGIL
-    struct type_cache type_cache;
-#endif
     size_t num_builtins_initialized;
     static_builtin_state builtins[_Py_MAX_STATIC_BUILTIN_TYPES];
 };

Makefile.pre.in

@@ -400,6 +400,7 @@ PYTHON_OBJS=	\
 		Python/lock.o \
 		Python/marshal.o \
 		Python/modsupport.o \
+		Python/mrocache.o \
 		Python/mysnprintf.o \
 		Python/mystrtoul.o \
 		Python/parking_lot.o \
@@ -1693,6 +1694,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/internal/pycore_list.h \
 		$(srcdir)/Include/internal/pycore_long.h \
 		$(srcdir)/Include/internal/pycore_moduleobject.h \
+		$(srcdir)/Include/internal/pycore_mrocache.h \
 		$(srcdir)/Include/internal/pycore_namespace.h \
 		$(srcdir)/Include/internal/pycore_object.h \
 		$(srcdir)/Include/internal/pycore_obmalloc.h \

Modules/_testbuffer.c

@@ -2825,6 +2825,9 @@ PyInit__testbuffer(void)
 {
     PyObject *m;
 
+    if (PyType_Ready(&NDArray_Type) < 0)
+        return NULL;
+
     m = PyModule_Create(&_testbuffermodule);
     if (m == NULL)
         return NULL;

Modules/gcmodule.c

@@ -33,6 +33,7 @@
 #include "pycore_pymem.h"
 #include "pycore_pystate.h"
 #include "pycore_refcnt.h"
+#include "pycore_qsbr.h"
 #include "pycore_gc.h"
 #include "frameobject.h"        /* for PyFrame_ClearFreeList */
 #include "pydtrace.h"
@@ -1697,6 +1698,10 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason)
      */
     handle_legacy_finalizers(tstate, gcstate, &finalizers);
 
+    _Py_qsbr_advance(&_PyRuntime.qsbr_shared);
+    _Py_qsbr_quiescent_state(tstate);
+    _PyMem_QsbrPoll(tstate);
+
     if (_PyErr_Occurred(tstate)) {
         if (reason == GC_REASON_SHUTDOWN) {
             _PyErr_Clear(tstate);