bpo-46417: remove_subclass() clears tp_subclasses (GH-30793)

The remove_subclass() function now deletes the dictionary when
removing the last subclass (if the dictionary becomes empty) to save
memory: set PyTypeObject.tp_subclasses to NULL. remove_subclass() is
called when a type is deallocated.

_PyType_GetSubclasses() no longer holds a reference to tp_subclasses:
its loop cannot modify tp_subclasses.

Author: vstinner

Lib/test/test_descr.py

@@ -4923,6 +4923,23 @@ def __new__(cls):
                 cls.lst = [2**i for i in range(10000)]
         X.descr
 
+    def test_remove_subclass(self):
+        # bpo-46417: when the last subclass of a type is deleted,
+        # remove_subclass() clears the internal dictionary of subclasses:
+        # set PyTypeObject.tp_subclasses to NULL. remove_subclass() is called
+        # when a type is deallocated.
+        class Parent:
+            pass
+        self.assertEqual(Parent.__subclasses__(), [])
+
+        class Child(Parent):
+            pass
+        self.assertEqual(Parent.__subclasses__(), [Child])
+
+        del Child
+        gc.collect()
+        self.assertEqual(Parent.__subclasses__(), [])
+
 
 class DictProxyTests(unittest.TestCase):
     def setUp(self):

Objects/typeobject.c

@@ -4137,29 +4137,28 @@ _PyType_GetSubclasses(PyTypeObject *self)
         return NULL;
     }
 
-    // Hold a strong reference to tp_subclasses while iterating on it
-    PyObject *dict = Py_XNewRef(self->tp_subclasses);
-    if (dict == NULL) {
+    PyObject *subclasses = self->tp_subclasses;  // borrowed ref
+    if (subclasses == NULL) {
         return list;
     }
-    assert(PyDict_CheckExact(dict));
+    assert(PyDict_CheckExact(subclasses));
+    // The loop cannot modify tp_subclasses, there is no need
+    // to hold a strong reference (use a borrowed reference).
 
     Py_ssize_t i = 0;
     PyObject *ref;  // borrowed ref
-    while (PyDict_Next(dict, &i, NULL, &ref)) {
+    while (PyDict_Next(subclasses, &i, NULL, &ref)) {
         assert(PyWeakref_CheckRef(ref));
         PyObject *obj = PyWeakref_GET_OBJECT(ref);  // borrowed ref
         if (obj == Py_None) {
             continue;
         }
         assert(PyType_Check(obj));
         if (PyList_Append(list, obj) < 0) {
-            Py_CLEAR(list);
-            goto done;
+            Py_DECREF(list);
+            return NULL;
         }
     }
-done:
-    Py_DECREF(dict);
     return list;
 }
 
@@ -6568,6 +6567,13 @@ remove_subclass(PyTypeObject *base, PyTypeObject *type)
         PyErr_Clear();
     }
     Py_XDECREF(key);
+
+    if (PyDict_Size(dict) == 0) {
+        // Delete the dictionary to save memory. _PyStaticType_Dealloc()
+        // callers also test if tp_subclasses is NULL to check if a static type
+        // has no subclass.
+        Py_CLEAR(base->tp_subclasses);
+    }
 }
 
 static void