bpo-46541: Replace core use of _Py_IDENTIFIER() with statically initialized global objects. (gh-30928)

We're no longer using _Py_IDENTIFIER() (or _Py_static_string()) in any core CPython code.  It is still used in a number of non-builtin stdlib modules.

The replacement is: PyUnicodeObject (not pointer) fields under _PyRuntimeState, statically initialized as part of _PyRuntime.  A new _Py_GET_GLOBAL_IDENTIFIER() macro facilitates lookup of the fields (along with _Py_GET_GLOBAL_STRING() for non-identifier strings).

https://bugs.python.org/issue46541#msg411799 explains the rationale for this change.

The core of the change is in:

* (new) Include/internal/pycore_global_strings.h - the declarations for the global strings, along with the macros
* Include/internal/pycore_runtime_init.h - added the static initializers for the global strings
* Include/internal/pycore_global_objects.h - where the struct in pycore_global_strings.h is hooked into _PyRuntimeState
* Tools/scripts/generate_global_objects.py - added generation of the global string declarations and static initializers

I've also added a --check flag to generate_global_objects.py (along with make check-global-objects) to check for unused global strings.  That check is added to the PR CI config.

The remainder of this change updates the core code to use _Py_GET_GLOBAL_IDENTIFIER() instead of _Py_IDENTIFIER() and the related _Py*Id functions (likewise for _Py_GET_GLOBAL_STRING() instead of _Py_static_string()).  This includes adding a few functions where there wasn't already an alternative to _Py*Id(), replacing the _Py_Identifier * parameter with PyObject *.

The following are not changed (yet):

* stop using _Py_IDENTIFIER() in the stdlib modules
* (maybe) get rid of _Py_IDENTIFIER(), etc. entirely -- this may not be doable as at least one package on PyPI using this (private) API
* (maybe) intern the strings during runtime init

https://bugs.python.org/issue46541

Author: ericsnowcurrently

.github/workflows/build.yml

@@ -100,6 +100,8 @@ jobs:
         run: make smelly
       - name: Check limited ABI symbols
         run: make check-limited-abi
+      - name: Check global objects
+        run: make check-global-objects
 
   build_win32:
     name: 'Windows (x86)'

Include/cpython/abstract.h

@@ -116,6 +116,10 @@ PyObject_CallMethodOneArg(PyObject *self, PyObject *name, PyObject *arg)
            2 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
 }
 
+PyAPI_FUNC(PyObject *) _PyObject_CallMethod(PyObject *obj,
+                                            PyObject *name,
+                                            const char *format, ...);
+
 /* Like PyObject_CallMethod(), but expect a _Py_Identifier*
    as the method name. */
 PyAPI_FUNC(PyObject *) _PyObject_CallMethodId(PyObject *obj,

Include/cpython/ceval.h

@@ -15,6 +15,7 @@ PyAPI_FUNC(int) _PyEval_SetAsyncGenFinalizer(PyObject *);
 PyAPI_FUNC(PyObject *) _PyEval_GetAsyncGenFinalizer(void);
 
 /* Helper to look up a builtin object */
+PyAPI_FUNC(PyObject *) _PyEval_GetBuiltin(PyObject *);
 PyAPI_FUNC(PyObject *) _PyEval_GetBuiltinId(_Py_Identifier *);
 /* Look at the current frame's (if any) code's co_flags, and turn on
    the corresponding compiler flags in cf->cf_flags.  Return 1 if any

Include/cpython/dictobject.h

@@ -30,6 +30,7 @@ typedef struct {
 
 PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key,
                                        Py_hash_t hash);
+PyAPI_FUNC(PyObject *) _PyDict_GetItemWithError(PyObject *dp, PyObject *key);
 PyAPI_FUNC(PyObject *) _PyDict_GetItemIdWithError(PyObject *dp,
                                                   struct _Py_Identifier *key);
 PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObject *, const char *);

Include/cpython/object.h

@@ -43,10 +43,16 @@ typedef struct _Py_Identifier {
     Py_ssize_t index;
 } _Py_Identifier;
 
+#if defined(NEEDS_PY_IDENTIFIER) || !defined(Py_BUILD_CORE)
+// For now we are keeping _Py_IDENTIFIER for continued use
+// in non-builtin extensions (and naughty PyPI modules).
+
 #define _Py_static_string_init(value) { .string = value, .index = -1 }
 #define _Py_static_string(varname, value)  static _Py_Identifier varname = _Py_static_string_init(value)
 #define _Py_IDENTIFIER(varname) _Py_static_string(PyId_##varname, #varname)
 
+#endif  /* NEEDS_PY_IDENTIFIER */
+
 typedef int (*getbufferproc)(PyObject *, Py_buffer *, int);
 typedef void (*releasebufferproc)(PyObject *, Py_buffer *);
 
@@ -249,7 +255,12 @@ typedef struct _heaptypeobject {
 PyAPI_FUNC(const char *) _PyType_Name(PyTypeObject *);
 PyAPI_FUNC(PyObject *) _PyType_Lookup(PyTypeObject *, PyObject *);
 PyAPI_FUNC(PyObject *) _PyType_LookupId(PyTypeObject *, _Py_Identifier *);
-PyAPI_FUNC(PyObject *) _PyObject_LookupSpecial(PyObject *, _Py_Identifier *);
+PyAPI_FUNC(PyObject *) _PyObject_LookupSpecialId(PyObject *, _Py_Identifier *);
+#ifndef Py_BUILD_CORE
+// Backward compatibility for 3rd-party extensions
+// that may be using the old name.
+#define _PyObject_LookupSpecial _PyObject_LookupSpecialId
+#endif
 PyAPI_FUNC(PyTypeObject *) _PyType_CalculateMetaclass(PyTypeObject *, PyObject *);
 PyAPI_FUNC(PyObject *) _PyType_GetDocFromInternalDoc(const char *, const char *);
 PyAPI_FUNC(PyObject *) _PyType_GetTextSignatureFromInternalDoc(const char *, const char *);

Include/cpython/sysmodule.h

@@ -2,6 +2,8 @@
 #  error "this header file must not be included directly"
 #endif
 
+PyAPI_FUNC(PyObject *) _PySys_GetAttr(PyThreadState *tstate,
+                                      PyObject *name);
 PyAPI_FUNC(PyObject *) _PySys_GetObjectId(_Py_Identifier *key);
 PyAPI_FUNC(int) _PySys_SetObjectId(_Py_Identifier *key, PyObject *);
 

Include/internal/pycore_call.h

@@ -30,6 +30,9 @@ PyAPI_FUNC(PyObject *) _PyObject_Call(
     PyObject *args,
     PyObject *kwargs);
 
+extern PyObject * _PyObject_CallMethodFormat(
+        PyThreadState *tstate, PyObject *callable, const char *format, ...);
+
 
 // Static inline variant of public PyVectorcall_Function().
 static inline vectorcallfunc

Include/internal/pycore_global_objects.h

@@ -8,6 +8,8 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_global_strings.h"  // struct _Py_global_strings
+
 
 // These would be in pycore_long.h if it weren't for an include cycle.
 #define _PY_NSMALLPOSINTS           257
@@ -36,6 +38,8 @@ struct _Py_global_objects {
             PyBytesObject ob;
             char eos;
         } bytes_characters[256];
+
+        struct _Py_global_strings strings;
     } singletons;
 };