bpo-36763: PyConfig_Read() handles PySys_AddXOption() (GH-15431)

PyConfig_Read() is now responsible to handle early calls to
PySys_AddXOption() and PySys_AddWarnOption().

Options added by PySys_AddXOption() are now handled the same way than
PyConfig.xoptions and command line -X options.

For example, PySys_AddXOption(L"faulthandler") enables faulthandler
as expected.

Author: vstinner

Include/internal/pycore_pylifecycle.h

@@ -43,6 +43,8 @@ extern PyStatus _PySys_Create(
     PyThreadState *tstate,
     PyObject **sysmod_p);
 extern PyStatus _PySys_SetPreliminaryStderr(PyObject *sysdict);
+extern PyStatus _PySys_ReadPreinitWarnOptions(PyConfig *config);
+extern PyStatus _PySys_ReadPreinitXOptions(PyConfig *config);
 extern int _PySys_InitMain(
     _PyRuntimeState *runtime,
     PyThreadState *tstate);

Lib/test/test_embed.py

@@ -902,6 +902,23 @@ def modify_path(path):
                                api=API_PYTHON,
                                modify_path_cb=modify_path)
 
+    def test_init_sys_add(self):
+        config = {
+            'faulthandler': 1,
+            'xoptions': [
+                'config_xoption',
+                'cmdline_xoption',
+                'sysadd_xoption',
+                'faulthandler',
+            ],
+            'warnoptions': [
+                'ignore:::config_warnoption',
+                'ignore:::cmdline_warnoption',
+                'ignore:::sysadd_warnoption',
+            ],
+        }
+        self.check_all_configs("test_init_sys_add", config, api=API_PYTHON)
+
     def test_init_run_main(self):
         code = ('import _testinternalcapi, json; '
                 'print(json.dumps(_testinternalcapi.get_configs()))')

Misc/NEWS.d/next/C API/2019-08-23-18-45-11.bpo-36763.q3Kh8Z.rst

@@ -0,0 +1,2 @@
+Options added by ``PySys_AddXOption()`` are now handled the same way than
+``PyConfig.xoptions`` and command line ``-X`` options.

Programs/_testembed.c

@@ -1375,6 +1375,54 @@ static int test_init_read_set(void)
 }
 
 
+static int test_init_sys_add(void)
+{
+    PySys_AddXOption(L"sysadd_xoption");
+    PySys_AddXOption(L"faulthandler");
+    PySys_AddWarnOption(L"ignore:::sysadd_warnoption");
+
+    PyConfig config;
+    PyStatus status;
+    status = PyConfig_InitPythonConfig(&config);
+    if (PyStatus_Exception(status)) {
+        goto fail;
+    }
+
+    wchar_t* argv[] = {
+        L"python3",
+        L"-W",
+        L"ignore:::cmdline_warnoption",
+        L"-X",
+        L"cmdline_xoption",
+    };
+    config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv);
+    config.parse_argv = 1;
+
+    status = PyWideStringList_Append(&config.xoptions,
+                                     L"config_xoption");
+    if (PyStatus_Exception(status)) {
+        goto fail;
+    }
+
+    status = PyWideStringList_Append(&config.warnoptions,
+                                     L"ignore:::config_warnoption");
+    if (PyStatus_Exception(status)) {
+        goto fail;
+    }
+
+    config_set_program_name(&config);
+    init_from_config_clear(&config);
+
+    dump_config();
+    Py_Finalize();
+    return 0;
+
+fail:
+    PyConfig_Clear(&config);
+    Py_ExitStatusException(status);
+}
+
+
 static void configure_init_main(PyConfig *config)
 {
     wchar_t* argv[] = {
@@ -1510,6 +1558,7 @@ static struct TestCase TestCases[] = {
     {"test_init_read_set", test_init_read_set},
     {"test_init_run_main", test_init_run_main},
     {"test_init_main", test_init_main},
+    {"test_init_sys_add", test_init_sys_add},
     {"test_run_main", test_run_main},
     {"test_open_code_hook", test_open_code_hook},
     {"test_audit", test_audit},

Python/initconfig.c

@@ -2069,6 +2069,7 @@ config_init_warnoptions(PyConfig *config,
     /* The priority order for warnings configuration is (highest precedence
      * first):
      *
+     * - early PySys_AddWarnOption() calls
      * - the BytesWarning filter, if needed ('-b', '-bb')
      * - any '-W' command line options; then
      * - the 'PYTHONWARNINGS' environment variable; then
@@ -2124,6 +2125,13 @@ config_init_warnoptions(PyConfig *config,
             return status;
         }
     }
+
+    /* Handle early PySys_AddWarnOption() calls */
+    status = _PySys_ReadPreinitWarnOptions(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+
     return _PyStatus_OK();
 }
 
@@ -2293,7 +2301,8 @@ config_read_cmdline(PyConfig *config)
     }
 
     status = config_init_warnoptions(config,
-                                  &cmdline_warnoptions, &env_warnoptions);
+                                     &cmdline_warnoptions,
+                                     &env_warnoptions);
     if (_PyStatus_EXCEPTION(status)) {
         goto done;
     }
@@ -2403,6 +2412,12 @@ PyConfig_Read(PyConfig *config)
         goto done;
     }
 
+    /* Handle early PySys_AddXOption() calls */
+    status = _PySys_ReadPreinitXOptions(config);
+    if (_PyStatus_EXCEPTION(status)) {
+        goto done;
+    }
+
     status = config_read(config);
     if (_PyStatus_EXCEPTION(status)) {
         goto done;

Python/sysmodule.c

@@ -2037,36 +2037,43 @@ _clear_preinit_entries(_Py_PreInitEntry *optionlist)
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 }
 
-static void
-_clear_all_preinit_options(void)
+
+PyStatus
+_PySys_ReadPreinitWarnOptions(PyConfig *config)
 {
+    PyStatus status;
+    _Py_PreInitEntry entry;
+
+    for (entry = _preinit_warnoptions; entry != NULL; entry = entry->next) {
+        status = PyWideStringList_Append(&config->warnoptions, entry->value);
+        if (_PyStatus_EXCEPTION(status)) {
+            return status;
+        }
+    }
+
     _clear_preinit_entries(&_preinit_warnoptions);
-    _clear_preinit_entries(&_preinit_xoptions);
+    return _PyStatus_OK();
 }
 
-static int
-sys_read_preinit_options(PyThreadState *tstate)
+
+PyStatus
+_PySys_ReadPreinitXOptions(PyConfig *config)
 {
-    /* Rerun the add commands with the actual sys module available */
-    if (tstate == NULL) {
-        /* Still don't have a thread state, so something is wrong! */
-        return -1;
-    }
-    _Py_PreInitEntry entry = _preinit_warnoptions;
-    while (entry != NULL) {
-        PySys_AddWarnOption(entry->value);
-        entry = entry->next;
-    }
-    entry = _preinit_xoptions;
-    while (entry != NULL) {
-        PySys_AddXOption(entry->value);
-        entry = entry->next;
+    PyStatus status;
+    _Py_PreInitEntry entry;
+
+    for (entry = _preinit_xoptions; entry != NULL; entry = entry->next) {
+        status = PyWideStringList_Append(&config->xoptions, entry->value);
+        if (_PyStatus_EXCEPTION(status)) {
+            return status;
+        }
     }
 
-    _clear_all_preinit_options();
-    return 0;
+    _clear_preinit_entries(&_preinit_xoptions);
+    return _PyStatus_OK();
 }
 
+
 static PyObject *
 get_warnoptions(PyThreadState *tstate)
 {
@@ -2235,9 +2242,7 @@ PySys_AddXOption(const wchar_t *s)
     }
     if (_PySys_AddXOptionWithError(s) < 0) {
         /* No return value, therefore clear error state if possible */
-        if (tstate) {
-            _PyErr_Clear(tstate);
-        }
+        _PyErr_Clear(tstate);
     }
 }
 
@@ -2898,11 +2903,6 @@ _PySys_InitMain(_PyRuntimeState *runtime, PyThreadState *tstate)
     if (get_xoptions(tstate) == NULL)
         return -1;
 
-    /* Transfer any sys.warnoptions and sys._xoptions set directly
-     * by an embedding application from the linked list to the module. */
-    if (sys_read_preinit_options(tstate) != 0)
-        return -1;
-
     if (_PyErr_Occurred(tstate)) {
         goto err_occurred;
     }