bpo-34485, Windows: LC_CTYPE set to user preference (GH-8988)

On Windows, the LC_CTYPE is now set to the user preferred locale at
startup: _Py_SetLocaleFromEnv(LC_CTYPE) is now called during the
Python initialization. Previously, the LC_CTYPE locale was "C" at
startup, but changed when calling setlocale(LC_CTYPE, "") or
setlocale(LC_ALL, "").

pymain_read_conf() now also calls _Py_SetLocaleFromEnv(LC_CTYPE) to
behave as _Py_InitializeCore(). Moreover, it doesn't save/restore the
LC_ALL anymore.

On Windows, standard streams like sys.stdout now always use
surrogateescape error handler by default (ignore the locale).

Author: vstinner

Misc/NEWS.d/next/Core and Builtins/2018-08-29-11-04-19.bpo-34485.c2AFdp.rst

@@ -0,0 +1,3 @@
+On Windows, the LC_CTYPE is now set to the user preferred locale at startup.
+Previously, the LC_CTYPE locale was "C" at startup, but changed when calling
+setlocale(LC_CTYPE, "") or setlocale(LC_ALL, "").

Modules/main.c

@@ -1280,25 +1280,18 @@ pymain_read_conf_impl(_PyMain *pymain, _PyCoreConfig *config,
 }
 
 
-/* Read the configuration, but initialize also the LC_CTYPE locale:
-   enable UTF-8 mode (PEP 540) and/or coerce the C locale (PEP 538) */
+/* Read the configuration and initialize the LC_CTYPE locale:
+   enable UTF-8 mode (PEP 540) and/or coerce the C locale (PEP 538). */
 static int
 pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config,
                  _PyCmdline *cmdline)
 {
     int init_utf8_mode = Py_UTF8Mode;
     _PyCoreConfig save_config = _PyCoreConfig_INIT;
-    char *oldloc = NULL;
     int res = -1;
 
-    oldloc = _PyMem_RawStrdup(setlocale(LC_ALL, NULL));
-    if (oldloc == NULL) {
-        pymain->err = _Py_INIT_NO_MEMORY();
-        goto done;
-    }
-
-    /* Reconfigure the locale to the default for this process */
-    _Py_SetLocaleFromEnv(LC_ALL);
+    /* Set LC_CTYPE to the user preferred locale */
+    _Py_SetLocaleFromEnv(LC_CTYPE);
 
     int locale_coerced = 0;
     int loops = 0;
@@ -1386,10 +1379,6 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config,
 
 done:
     _PyCoreConfig_Clear(&save_config);
-    if (oldloc != NULL) {
-        setlocale(LC_ALL, oldloc);
-        PyMem_RawFree(oldloc);
-    }
     Py_UTF8Mode = init_utf8_mode ;
     return res;
 }

Python/pylifecycle.c

@@ -343,6 +343,7 @@ static _LocaleCoercionTarget _TARGET_LOCALES[] = {
 static const char *
 get_stdio_errors(void)
 {
+#ifndef MS_WINDOWS
     const char *ctype_loc = setlocale(LC_CTYPE, NULL);
     if (ctype_loc != NULL) {
         /* surrogateescape is the default in the legacy C and POSIX locales */
@@ -362,6 +363,10 @@ get_stdio_errors(void)
     }
 
     return "strict";
+#else
+    /* On Windows, always use surrogateescape by default */
+    return "surrogateescape";
+#endif
 }
 
 #ifdef PY_COERCE_C_LOCALE
@@ -751,11 +756,8 @@ _Py_InitializeCore(PyInterpreterState **interp_p,
        (and the input configuration is read only). */
     _PyCoreConfig config = _PyCoreConfig_INIT;
 
-#ifndef MS_WINDOWS
-    /* Set up the LC_CTYPE locale, so we can obtain the locale's charset
-       without having to switch locales. */
+    /* Set LC_CTYPE to the user preferred locale */
     _Py_SetLocaleFromEnv(LC_CTYPE);
-#endif
 
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
     if (_PyCoreConfig_Copy(&config, src_config) >= 0) {