[3.11] gh-99952: fix refcount issues in ctypes.Structure from_param() result (#101339)

[3.11] gh-99952: [ctypes] fix refcount issues in from_param() result. (GH-100169)

Fixes a reference counting issue with `ctypes.Structure` when a `from_param()` method call is used and the structure size is larger than a C pointer `sizeof(void*)`.

This problem existed for a very long time, but became more apparent in 3.8+ by change likely due to garbage collection cleanup timing changes..
(cherry picked from commit dfad678)

Co-authored-by: Yukihiro Nakadaira <[email protected]>

Author: gpshead

Lib/ctypes/test/test_parameters.py

@@ -244,6 +244,58 @@ def test_parameter_repr(self):
         self.assertRegex(repr(c_wchar_p.from_param('hihi')), r"^<cparam 'Z' \(0x[A-Fa-f0-9]+\)>$")
         self.assertRegex(repr(c_void_p.from_param(0x12)), r"^<cparam 'P' \(0x0*12\)>$")
 
+    @test.support.cpython_only
+    def test_from_param_result_refcount(self):
+        # Issue #99952
+        import _ctypes_test
+        from ctypes import PyDLL, c_int, c_void_p, py_object, Structure
+
+        class X(Structure):
+            """This struct size is <= sizeof(void*)."""
+            _fields_ = [("a", c_void_p)]
+
+            def __del__(self):
+                trace.append(4)
+
+            @classmethod
+            def from_param(cls, value):
+                trace.append(2)
+                return cls()
+
+        PyList_Append = PyDLL(_ctypes_test.__file__)._testfunc_pylist_append
+        PyList_Append.restype = c_int
+        PyList_Append.argtypes = [py_object, py_object, X]
+
+        trace = []
+        trace.append(1)
+        PyList_Append(trace, 3, "dummy")
+        trace.append(5)
+
+        self.assertEqual(trace, [1, 2, 3, 4, 5])
+
+        class Y(Structure):
+            """This struct size is > sizeof(void*)."""
+            _fields_ = [("a", c_void_p), ("b", c_void_p)]
+
+            def __del__(self):
+                trace.append(4)
+
+            @classmethod
+            def from_param(cls, value):
+                trace.append(2)
+                return cls()
+
+        PyList_Append = PyDLL(_ctypes_test.__file__)._testfunc_pylist_append
+        PyList_Append.restype = c_int
+        PyList_Append.argtypes = [py_object, py_object, Y]
+
+        trace = []
+        trace.append(1)
+        PyList_Append(trace, 3, "dummy")
+        trace.append(5)
+
+        self.assertEqual(trace, [1, 2, 3, 4, 5])
+
 ################################################################
 
 if __name__ == '__main__':

Misc/NEWS.d/next/Library/2022-12-11-14-38-59.gh-issue-99952.IYGLzr.rst

@@ -0,0 +1,2 @@
+Fix a reference undercounting issue in :class:`ctypes.Structure` with ``from_param()``
+results larger than a C pointer.

Modules/_ctypes/_ctypes.c

@@ -416,13 +416,15 @@ _ctypes_alloc_format_string_with_shape(int ndim, const Py_ssize_t *shape,
 typedef struct {
     PyObject_HEAD
     void *ptr;
+    PyObject *keep;  // If set, a reference to the original CDataObject.
 } StructParamObject;
 
 
 static void
 StructParam_dealloc(PyObject *myself)
 {
     StructParamObject *self = (StructParamObject *)myself;
+    Py_XDECREF(self->keep);
     PyMem_Free(self->ptr);
     Py_TYPE(self)->tp_free(myself);
 }
@@ -470,6 +472,7 @@ StructUnionType_paramfunc(CDataObject *self)
 
         StructParamObject *struct_param = (StructParamObject *)obj;
         struct_param->ptr = ptr;
+        struct_param->keep = Py_NewRef(self);
     } else {
         ptr = self->b_ptr;
         obj = (PyObject *)self;

Modules/_ctypes/_ctypes_test.c

@@ -1034,6 +1034,12 @@ EXPORT (HRESULT) KeepObject(IUnknown *punk)
 
 #endif
 
+EXPORT(int)
+_testfunc_pylist_append(PyObject *list, PyObject *item)
+{
+    return PyList_Append(list, item);
+}
+
 static struct PyModuleDef_Slot _ctypes_test_slots[] = {
     {0, NULL}
 };