bpo-36763: Add _PyCoreConfig_InitPythonConfig() (GH-13388)

Add new functions to get the Python interpreter behavior:

* _PyPreConfig_InitPythonConfig()
* _PyCoreConfig_InitPythonConfig()

Add new functions to get an isolated configuration:

* _PyPreConfig_InitIsolatedConfig()
* _PyCoreConfig_InitIsolatedConfig()

Replace _PyPreConfig_INIT and _PyCoreConfig_INIT with new functions
_PyPreConfig_Init() and _PyCoreConfig_Init().

_PyCoreConfig: set configure_c_stdio and parse_argv to 0 by default
to behave as Python 3.6 in the default configuration.

_PyCoreConfig_Read() no longer sets coerce_c_locale_warn to 1 if it's
equal to 0. coerce_c_locale_warn must now be set to -1 (ex: using
_PyCoreConfig_InitPythonConfig()) to enable C locale coercion
warning.

Add unit tests for _PyCoreConfig_InitPythonConfig()
and _PyCoreConfig_InitIsolatedConfig().

Changes:

* Rename _PyCoreConfig_GetCoreConfig() to _PyPreConfig_GetCoreConfig()
* Fix core_read_precmdline(): handle parse_argv=0
* Fix _Py_PreInitializeFromCoreConfig(): pass coreconfig.argv
  to _Py_PreInitializeFromPyArgv(), except if parse_argv=0

Author: vstinner

Include/cpython/coreconfig.h

@@ -86,12 +86,18 @@ typedef struct {
 
        If it is equal to 1, LC_CTYPE locale is read to decide it it should be
        coerced or not (ex: PYTHONCOERCECLOCALE=1). Internally, it is set to 2
-       if the LC_CTYPE locale must be coerced. */
+       if the LC_CTYPE locale must be coerced.
+
+       Disable by default (set to 0). Set it to -1 to let Python decides if it
+       should be enabled or not. */
     int coerce_c_locale;
 
     /* Emit a warning if the LC_CTYPE locale is coerced?
 
-       Disabled by default. Set to 1 by PYTHONCOERCECLOCALE=warn. */
+       Set to 1 by PYTHONCOERCECLOCALE=warn.
+
+       Disable by default (set to 0). Set it to -1 to let Python decides if it
+       should be enabled or not. */
     int coerce_c_locale_warn;
 
 #ifdef MS_WINDOWS
@@ -116,7 +122,10 @@ typedef struct {
        Set to 0 by "-X utf8=0" and PYTHONUTF8=0.
 
        If equals to -1, it is set to 1 if the LC_CTYPE locale is "C" or
-       "POSIX", otherwise inherit Py_UTF8Mode value. */
+       "POSIX", otherwise it is set to 0.
+
+       If equals to -2, inherit Py_UTF8Mode value value (which is equal to 0
+       by default). */
     int utf8_mode;
 
     int dev_mode;           /* Development mode. PYTHONDEVMODE, -X dev */
@@ -138,9 +147,14 @@ typedef struct {
         ._config_version = _Py_CONFIG_VERSION, \
         .isolated = -1, \
         .use_environment = -1, \
+        .utf8_mode = -2, \
         .dev_mode = -1, \
         .allocator = PYMEM_ALLOCATOR_NOT_SET}
 
+PyAPI_FUNC(void) _PyPreConfig_Init(_PyPreConfig *config);
+PyAPI_FUNC(void) _PyPreConfig_InitPythonConfig(_PyPreConfig *config);
+PyAPI_FUNC(void) _PyPreConfig_InitIsolateConfig(_PyPreConfig *config);
+
 
 /* --- _PyCoreConfig ---------------------------------------------- */
 
@@ -213,8 +227,8 @@ typedef struct {
 
     /* Command line arguments (sys.argv).
 
-       By default, Python command line arguments are parsed and then stripped
-       from argv. Set parse_argv to 0 to avoid that.
+       Set parse_argv to 1 to parse argv as Python command line arguments
+       and then strip Python arguments from argv.
 
        If argv is empty, an empty string is added to ensure that sys.argv
        always exists and is never empty. */
@@ -442,7 +456,7 @@ typedef struct {
         .faulthandler = -1, \
         .tracemalloc = -1, \
         .use_module_search_paths = 0, \
-        .parse_argv = 1, \
+        .parse_argv = 0, \
         .site_import = -1, \
         .bytes_warning = -1, \
         .inspect = -1, \
@@ -453,14 +467,18 @@ typedef struct {
         .verbose = -1, \
         .quiet = -1, \
         .user_site_directory = -1, \
-        .configure_c_stdio = 1, \
+        .configure_c_stdio = 0, \
         .buffered_stdio = -1, \
         ._install_importlib = 1, \
         .check_hash_pycs_mode = NULL, \
         .pathconfig_warnings = -1, \
         ._init_main = 1}
 /* Note: _PyCoreConfig_INIT sets other fields to 0/NULL */
 
+PyAPI_FUNC(void) _PyCoreConfig_Init(_PyCoreConfig *config);
+PyAPI_FUNC(_PyInitError) _PyCoreConfig_InitPythonConfig(_PyCoreConfig *config);
+PyAPI_FUNC(_PyInitError) _PyCoreConfig_InitIsolateConfig(_PyCoreConfig *config);
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_coreconfig.h

@@ -88,10 +88,13 @@ PyAPI_FUNC(_PyInitError) _PyPreCmdline_Read(_PyPreCmdline *cmdline,
 
 /* --- _PyPreConfig ----------------------------------------------- */
 
+PyAPI_FUNC(void) _PyPreConfig_Init(_PyPreConfig *config);
+PyAPI_FUNC(void) _PyPreConfig_InitPythonConfig(_PyPreConfig *config);
+PyAPI_FUNC(void) _PyPreConfig_InitIsolatedConfig(_PyPreConfig *config);
 PyAPI_FUNC(int) _PyPreConfig_Copy(_PyPreConfig *config,
     const _PyPreConfig *config2);
 PyAPI_FUNC(PyObject*) _PyPreConfig_AsDict(const _PyPreConfig *config);
-PyAPI_FUNC(void) _PyCoreConfig_GetCoreConfig(_PyPreConfig *config,
+PyAPI_FUNC(void) _PyPreConfig_GetCoreConfig(_PyPreConfig *config,
     const _PyCoreConfig *core_config);
 PyAPI_FUNC(_PyInitError) _PyPreConfig_Read(_PyPreConfig *config,
     const _PyArgv *args);
@@ -101,6 +104,8 @@ PyAPI_FUNC(_PyInitError) _PyPreConfig_Write(const _PyPreConfig *config);
 /* --- _PyCoreConfig ---------------------------------------------- */
 
 PyAPI_FUNC(void) _PyCoreConfig_Clear(_PyCoreConfig *);
+PyAPI_FUNC(_PyInitError) _PyCoreConfig_InitPythonConfig(_PyCoreConfig *config);
+PyAPI_FUNC(_PyInitError) _PyCoreConfig_InitIsolatedConfig(_PyCoreConfig *config);
 PyAPI_FUNC(_PyInitError) _PyCoreConfig_Copy(
     _PyCoreConfig *config,
     const _PyCoreConfig *config2);

Lib/test/test_embed.py

@@ -307,7 +307,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
 
         'pycache_prefix': None,
         'program_name': GET_DEFAULT_CONFIG,
-        'parse_argv': 1,
+        'parse_argv': 0,
         'argv': [""],
 
         'xoptions': [],
@@ -333,7 +333,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'verbose': 0,
         'quiet': 0,
         'user_site_directory': 1,
-        'configure_c_stdio': 1,
+        'configure_c_stdio': 0,
         'buffered_stdio': 1,
 
         'stdio_encoding': GET_DEFAULT_CONFIG,
@@ -588,6 +588,7 @@ def test_init_from_config(self):
             'pycache_prefix': 'conf_pycache_prefix',
             'program_name': './conf_program_name',
             'argv': ['-c', 'arg2'],
+            'parse_argv': 1,
             'xoptions': ['core_xoption1=3', 'core_xoption2=', 'core_xoption3'],
             'warnoptions': ['error::ResourceWarning', 'default::BytesWarning'],
             'run_command': 'pass\n',
@@ -600,7 +601,7 @@ def test_init_from_config(self):
             'write_bytecode': 0,
             'verbose': 1,
             'quiet': 1,
-            'configure_c_stdio': 0,
+            'configure_c_stdio': 1,
             'buffered_stdio': 0,
             'user_site_directory': 0,
             'faulthandler': 1,
@@ -661,14 +662,14 @@ def test_init_dev_mode(self):
         }
         self.check_config("init_dev_mode", config, preconfig)
 
-    def test_init_isolated(self):
+    def test_init_isolated_flag(self):
         preconfig = {}
         config = {
             'isolated': 1,
             'use_environment': 0,
             'user_site_directory': 0,
         }
-        self.check_config("init_isolated", config, preconfig)
+        self.check_config("init_isolated_flag", config, preconfig)
 
     def test_preinit_isolated1(self):
         # _PyPreConfig.isolated=1, _PyCoreConfig.isolated not set
@@ -690,6 +691,25 @@ def test_preinit_isolated2(self):
         }
         self.check_config("preinit_isolated2", config, preconfig)
 
+    def test_init_isolated_config(self):
+        preconfig = {}
+        config = {
+            'isolated': 1,
+            'use_environment': 0,
+            'user_site_directory': 0,
+            'install_signal_handlers': 0,
+            'pathconfig_warnings': 0,
+        }
+        self.check_config("init_isolated_config", config, preconfig)
+
+    def test_init_python_config(self):
+        preconfig = {}
+        config = {
+            'configure_c_stdio': 1,
+            'parse_argv': 1,
+        }
+        self.check_config("init_python_config", config, preconfig)
+
     def test_init_read_set(self):
         preconfig = {}
         core_config = {
@@ -707,6 +727,7 @@ def test_init_run_main(self):
             'argv': ['-c', 'arg2'],
             'program_name': './python3',
             'run_command': code + '\n',
+            'parse_argv': 1,
         }
         self.check_config("init_run_main", core_config, preconfig)
 
@@ -718,15 +739,26 @@ def test_init_main(self):
             'argv': ['-c', 'arg2'],
             'program_name': './python3',
             'run_command': code + '\n',
+            'parse_argv': 1,
             '_init_main': 0,
         }
         self.check_config("init_main", core_config, preconfig,
                           stderr="Run Python code before _Py_InitializeMain")
 
+    def test_init_parse_argv(self):
+        core_config = {
+            'argv': ['-c', 'arg1', '-v', 'arg3'],
+            'program_name': './argv0',
+            'parse_argv': 1,
+            'run_command': 'pass\n',
+            'use_environment': 0,
+        }
+        self.check_config("init_parse_argv", core_config, {})
+
     def test_init_dont_parse_argv(self):
         core_config = {
-            'argv': ['-v', '-c', 'arg1', '-W', 'arg2'],
-            'parse_argv': 0,
+            'argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'],
+            'program_name': './argv0',
         }
         self.check_config("init_dont_parse_argv", core_config, {})
 

Modules/main.c

@@ -52,23 +52,28 @@ pymain_init(const _PyArgv *args)
     fedisableexcept(FE_OVERFLOW);
 #endif
 
-    _PyPreConfig preconfig = _PyPreConfig_INIT;
-    /* Set to -1 to enable them depending on the LC_CTYPE locale and the
-       environment variables (PYTHONUTF8 and PYTHONCOERCECLOCALE)  */
-    preconfig.coerce_c_locale = -1;
-    preconfig.utf8_mode = -1;
+    _PyPreConfig preconfig;
+    _PyPreConfig_InitPythonConfig(&preconfig);
     err = _Py_PreInitializeFromPyArgv(&preconfig, args);
     if (_Py_INIT_FAILED(err)) {
         return err;
     }
 
+    _PyCoreConfig config;
+    err = _PyCoreConfig_InitPythonConfig(&config);
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+
     /* pass NULL as the config: config is read from command line arguments,
        environment variables, configuration files */
     if (args->use_bytes_argv) {
-        return _Py_InitializeFromArgs(NULL, args->argc, args->bytes_argv);
+        return _Py_InitializeFromArgs(&config,
+                                      args->argc, args->bytes_argv);
     }
     else {
-        return _Py_InitializeFromWideArgs(NULL, args->argc, args->wchar_argv);
+        return _Py_InitializeFromWideArgs(&config,
+                                          args->argc, args->wchar_argv);
     }
 }
 

Programs/_freeze_importlib.c

@@ -76,7 +76,8 @@ main(int argc, char *argv[])
     }
     text[text_size] = '\0';
 
-    _PyCoreConfig config = _PyCoreConfig_INIT;
+    _PyCoreConfig config;
+    _PyCoreConfig_Init(&config);
     config.use_environment = 0;
     config.user_site_directory = 0;
     config.site_import = 0;