bpo-42208: Add _locale._get_locale_encoding() (GH-23052)

* Add a new _locale._get_locale_encoding() function to get the
  current locale encoding.
* Modify locale.getpreferredencoding() to use it.
* Remove the _bootlocale module.

Author: vstinner

Lib/_bootlocale.py

@@ -619,53 +619,49 @@ def resetlocale(category=LC_ALL):
     """
     _setlocale(category, _build_localename(getdefaultlocale()))
 
-if sys.platform.startswith("win"):
-    # On Win32, this will return the ANSI code page
-    def getpreferredencoding(do_setlocale = True):
-        """Return the charset that the user is likely using."""
+
+try:
+    from _locale import _get_locale_encoding
+except ImportError:
+    def _get_locale_encoding():
+        if hasattr(sys, 'getandroidapilevel'):
+            # On Android langinfo.h and CODESET are missing, and UTF-8 is
+            # always used in mbstowcs() and wcstombs().
+            return 'UTF-8'
         if sys.flags.utf8_mode:
             return 'UTF-8'
-        import _bootlocale
-        return _bootlocale.getpreferredencoding(False)
+        encoding = getdefaultlocale()[1]
+        if encoding is None:
+            # LANG not set, default conservatively to ASCII
+            encoding = 'ascii'
+        return encoding
+
+try:
+    CODESET
+except NameError:
+    def getpreferredencoding(do_setlocale=True):
+        """Return the charset that the user is likely using."""
+        return _get_locale_encoding()
 else:
     # On Unix, if CODESET is available, use that.
-    try:
-        CODESET
-    except NameError:
-        if hasattr(sys, 'getandroidapilevel'):
-            # On Android langinfo.h and CODESET are missing, and UTF-8 is
-            # always used in mbstowcs() and wcstombs().
-            def getpreferredencoding(do_setlocale = True):
-                return 'UTF-8'
-        else:
-            # Fall back to parsing environment variables :-(
-            def getpreferredencoding(do_setlocale = True):
-                """Return the charset that the user is likely using,
-                by looking at environment variables."""
-                if sys.flags.utf8_mode:
-                    return 'UTF-8'
-                res = getdefaultlocale()[1]
-                if res is None:
-                    # LANG not set, default conservatively to ASCII
-                    res = 'ascii'
-                return res
-    else:
-        def getpreferredencoding(do_setlocale = True):
-            """Return the charset that the user is likely using,
-            according to the system configuration."""
-            if sys.flags.utf8_mode:
-                return 'UTF-8'
-            import _bootlocale
-            if do_setlocale:
-                oldloc = setlocale(LC_CTYPE)
-                try:
-                    setlocale(LC_CTYPE, "")
-                except Error:
-                    pass
-            result = _bootlocale.getpreferredencoding(False)
-            if do_setlocale:
-                setlocale(LC_CTYPE, oldloc)
-            return result
+    def getpreferredencoding(do_setlocale=True):
+        """Return the charset that the user is likely using,
+        according to the system configuration."""
+        if sys.flags.utf8_mode:
+            return 'UTF-8'
+
+        if not do_setlocale:
+            return _get_locale_encoding()
+
+        old_loc = setlocale(LC_CTYPE)
+        try:
+            try:
+                setlocale(LC_CTYPE, "")
+            except Error:
+                pass
+            return _get_locale_encoding()
+        finally:
+            setlocale(LC_CTYPE, old_loc)
 
 
 ### Database

Lib/locale.py

@@ -3,7 +3,7 @@
 import mimetypes
 import pathlib
 import sys
-import unittest
+import unittest.mock
 
 from test import support
 from test.support import os_helper
@@ -71,14 +71,14 @@ def test_read_mime_types(self):
         # bpo-41048: read_mime_types should read the rule file with 'utf-8' encoding.
         # Not with locale encoding. _bootlocale has been imported because io.open(...)
         # uses it.
-        with os_helper.temp_dir() as directory:
-            data = "application/no-mans-land  Fran\u00E7ais"
-            file = pathlib.Path(directory, "sample.mimetype")
-            file.write_text(data, encoding='utf-8')
-            import _bootlocale
-            with support.swap_attr(_bootlocale, 'getpreferredencoding', lambda do_setlocale=True: 'ASCII'):
-                mime_dict = mimetypes.read_mime_types(file)
-            eq(mime_dict[".Français"], "application/no-mans-land")
+        data = "application/no-mans-land  Fran\u00E7ais"
+        filename = "filename"
+        fp = io.StringIO(data)
+        with unittest.mock.patch.object(mimetypes, 'open',
+                                        return_value=fp) as mock_open:
+            mime_dict = mimetypes.read_mime_types(filename)
+            mock_open.assert_called_with(filename, encoding='utf-8')
+        eq(mime_dict[".Français"], "application/no-mans-land")
 
     def test_non_standard_types(self):
         eq = self.assertEqual

Lib/test/test_mimetypes.py

@@ -768,9 +768,24 @@ _locale_bind_textdomain_codeset_impl(PyObject *module, const char *domain,
     }
     Py_RETURN_NONE;
 }
-#endif
+#endif  // HAVE_BIND_TEXTDOMAIN_CODESET
+
+#endif  // HAVE_LIBINTL_H
+
+
+/*[clinic input]
+_locale._get_locale_encoding
+
+Get the current locale encoding.
+[clinic start generated code]*/
+
+static PyObject *
+_locale__get_locale_encoding_impl(PyObject *module)
+/*[clinic end generated code: output=e8e2f6f6f184591a input=513d9961d2f45c76]*/
+{
+    return _Py_GetLocaleEncoding();
+}
 
-#endif
 
 static struct PyMethodDef PyLocale_Methods[] = {
     _LOCALE_SETLOCALE_METHODDEF
@@ -797,6 +812,7 @@ static struct PyMethodDef PyLocale_Methods[] = {
     _LOCALE_BIND_TEXTDOMAIN_CODESET_METHODDEF
 #endif
 #endif
+    _LOCALE__GET_LOCALE_ENCODING_METHODDEF
   {NULL, NULL}
 };
 

Modules/_localemodule.c

@@ -1572,7 +1572,6 @@
     <Compile Include="zoneinfo\__init__.py" />
     <Compile Include="zoneinfo\_tzpath.py" />
     <Compile Include="zoneinfo\_zoneinfo.py" />
-    <Compile Include="_bootlocale.py" />
     <Compile Include="_collections_abc.py" />
     <Compile Include="_compat_pickle.py" />
     <Compile Include="_compression.py" />