bpo-39542: Make PyObject_INIT() opaque in limited C API (GH-18363)

In the limited C API, PyObject_INIT() and PyObject_INIT_VAR() are now
defined as aliases to PyObject_Init() and PyObject_InitVar() to make
their implementation opaque. It avoids to leak implementation details
in the limited C API.

Exclude the following functions from the limited C API, move them
from object.h to cpython/object.h:

* _Py_NewReference()
* _Py_ForgetReference()
* _PyTraceMalloc_NewReference()
* _Py_GetRefTotal()

Author: vstinner

Include/cpython/object.h

@@ -6,6 +6,22 @@
 extern "C" {
 #endif
 
+PyAPI_FUNC(void) _Py_NewReference(PyObject *op);
+
+#ifdef Py_TRACE_REFS
+/* Py_TRACE_REFS is such major surgery that we call external routines. */
+PyAPI_FUNC(void) _Py_ForgetReference(PyObject *);
+#endif
+
+/* Update the Python traceback of an object. This function must be called
+   when a memory block is reused from a free list. */
+PyAPI_FUNC(int) _PyTraceMalloc_NewReference(PyObject *op);
+
+#ifdef Py_REF_DEBUG
+PyAPI_FUNC(Py_ssize_t) _Py_GetRefTotal(void);
+#endif
+
+
 /********************* String Literals ****************************************/
 /* This structure helps managing static strings. The basic usage goes like this:
    Instead of doing

Include/cpython/objimpl.h

@@ -6,6 +6,39 @@
 extern "C" {
 #endif
 
+/* Inline functions trading binary compatibility for speed:
+   PyObject_INIT() is the fast version of PyObject_Init(), and
+   PyObject_INIT_VAR() is the fast version of PyObject_InitVar().
+
+   These inline functions must not be called with op=NULL. */
+static inline PyObject*
+_PyObject_INIT(PyObject *op, PyTypeObject *typeobj)
+{
+    assert(op != NULL);
+    Py_TYPE(op) = typeobj;
+    if (PyType_GetFlags(typeobj) & Py_TPFLAGS_HEAPTYPE) {
+        Py_INCREF(typeobj);
+    }
+    _Py_NewReference(op);
+    return op;
+}
+
+#define PyObject_INIT(op, typeobj) \
+    _PyObject_INIT(_PyObject_CAST(op), (typeobj))
+
+static inline PyVarObject*
+_PyObject_INIT_VAR(PyVarObject *op, PyTypeObject *typeobj, Py_ssize_t size)
+{
+    assert(op != NULL);
+    Py_SIZE(op) = size;
+    PyObject_INIT((PyObject *)op, typeobj);
+    return op;
+}
+
+#define PyObject_INIT_VAR(op, typeobj, size) \
+    _PyObject_INIT_VAR(_PyVarObject_CAST(op), (typeobj), (size))
+
+
 /* This function returns the number of allocated memory blocks, regardless of size */
 PyAPI_FUNC(Py_ssize_t) _Py_GetAllocatedBlocks(void);
 

Include/object.h

@@ -233,8 +233,7 @@ PyAPI_FUNC(int) PyObject_SetAttr(PyObject *, PyObject *, PyObject *);
 PyAPI_FUNC(int) PyObject_HasAttr(PyObject *, PyObject *);
 PyAPI_FUNC(PyObject *) PyObject_SelfIter(PyObject *);
 PyAPI_FUNC(PyObject *) PyObject_GenericGetAttr(PyObject *, PyObject *);
-PyAPI_FUNC(int) PyObject_GenericSetAttr(PyObject *,
-                                              PyObject *, PyObject *);
+PyAPI_FUNC(int) PyObject_GenericSetAttr(PyObject *, PyObject *, PyObject *);
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000
 PyAPI_FUNC(int) PyObject_GenericSetDict(PyObject *, PyObject *, void *);
 #endif
@@ -381,20 +380,8 @@ you can count such references to the type object.)
 PyAPI_DATA(Py_ssize_t) _Py_RefTotal;
 PyAPI_FUNC(void) _Py_NegativeRefcount(const char *filename, int lineno,
                                       PyObject *op);
-PyAPI_FUNC(Py_ssize_t) _Py_GetRefTotal(void);
 #endif /* Py_REF_DEBUG */
 
-/* Update the Python traceback of an object. This function must be called
-   when a memory block is reused from a free list. */
-PyAPI_FUNC(int) _PyTraceMalloc_NewReference(PyObject *op);
-
-PyAPI_FUNC(void) _Py_NewReference(PyObject *op);
-
-#ifdef Py_TRACE_REFS
-/* Py_TRACE_REFS is such major surgery that we call external routines. */
-PyAPI_FUNC(void) _Py_ForgetReference(PyObject *);
-#endif
-
 PyAPI_FUNC(void) _Py_Dealloc(PyObject *);
 
 static inline void _Py_INCREF(PyObject *op)

Include/objimpl.h

@@ -127,40 +127,21 @@ PyAPI_FUNC(PyVarObject *) _PyObject_NewVar(PyTypeObject *, Py_ssize_t);
 #define PyObject_NewVar(type, typeobj, n) \
                 ( (type *) _PyObject_NewVar((typeobj), (n)) )
 
-/* Inline functions trading binary compatibility for speed:
-   PyObject_INIT() is the fast version of PyObject_Init(), and
-   PyObject_INIT_VAR() is the fast version of PyObject_InitVar.
-   See also pymem.h.
-
-   These inline functions expect non-NULL object pointers. */
-static inline PyObject*
-_PyObject_INIT(PyObject *op, PyTypeObject *typeobj)
-{
-    assert(op != NULL);
-    Py_TYPE(op) = typeobj;
-    if (PyType_GetFlags(typeobj) & Py_TPFLAGS_HEAPTYPE) {
-        Py_INCREF(typeobj);
-    }
-    _Py_NewReference(op);
-    return op;
-}
-
-#define PyObject_INIT(op, typeobj) \
-    _PyObject_INIT(_PyObject_CAST(op), (typeobj))
+#define _PyObject_SIZE(typeobj) ( (typeobj)->tp_basicsize )
 
-static inline PyVarObject*
-_PyObject_INIT_VAR(PyVarObject *op, PyTypeObject *typeobj, Py_ssize_t size)
-{
-    assert(op != NULL);
-    Py_SIZE(op) = size;
-    PyObject_INIT((PyObject *)op, typeobj);
-    return op;
-}
 
-#define PyObject_INIT_VAR(op, typeobj, size) \
-    _PyObject_INIT_VAR(_PyVarObject_CAST(op), (typeobj), (size))
+#ifdef Py_LIMITED_API
+/* Define PyObject_INIT() and PyObject_INIT_VAR() as aliases to PyObject_Init()
+   and PyObject_InitVar() in the limited C API for compatibility with the
+   CPython C API. */
+#  define PyObject_INIT(op, typeobj) \
+        PyObject_Init(_PyObject_CAST(op), (typeobj))
+#  define PyObject_INIT_VAR(op, typeobj, size) \
+        PyObject_InitVar(_PyVarObject_CAST(op), (typeobj), (size))
+#else
+/* PyObject_INIT() and PyObject_INIT_VAR() are defined in cpython/objimpl.h */
+#endif
 
-#define _PyObject_SIZE(typeobj) ( (typeobj)->tp_basicsize )
 
 /* _PyObject_VAR_SIZE returns the number of bytes (as size_t) allocated for a
    vrbl-size object with nitems items, exclusive of gc overhead (if any).  The

Misc/NEWS.d/next/C API/2020-02-05-12-40-51.bpo-39542.si-_Zq.rst

@@ -0,0 +1,7 @@
+In the limited C API, ``PyObject_INIT()`` and ``PyObject_INIT_VAR()`` are
+now defined as aliases to :c:func:`PyObject_Init` and
+:c:func:`PyObject_InitVar` to make their implementation opaque. It avoids to
+leak implementation details in the limited C API. Exclude the following
+functions from the limited C API: ``_Py_NewReference()``,
+``_Py_ForgetReference()``, ``_PyTraceMalloc_NewReference()`` and
+``_Py_GetRefTotal()``.

Objects/object.c

@@ -139,9 +139,11 @@ Py_DecRef(PyObject *o)
 PyObject *
 PyObject_Init(PyObject *op, PyTypeObject *tp)
 {
-    if (op == NULL)
+    /* Any changes should be reflected in PyObject_INIT() macro */
+    if (op == NULL) {
         return PyErr_NoMemory();
-    /* Any changes should be reflected in PyObject_INIT (objimpl.h) */
+    }
+
     Py_TYPE(op) = tp;
     if (PyType_GetFlags(tp) & Py_TPFLAGS_HEAPTYPE) {
         Py_INCREF(tp);
@@ -153,9 +155,11 @@ PyObject_Init(PyObject *op, PyTypeObject *tp)
 PyVarObject *
 PyObject_InitVar(PyVarObject *op, PyTypeObject *tp, Py_ssize_t size)
 {
-    if (op == NULL)
+    /* Any changes should be reflected in PyObject_INIT_VAR() macro */
+    if (op == NULL) {
         return (PyVarObject *) PyErr_NoMemory();
-    /* Any changes should be reflected in PyObject_INIT_VAR */
+    }
+
     Py_SIZE(op) = size;
     PyObject_Init((PyObject *)op, tp);
     return op;