gh-119310: Fix encoding when reading old history file (#121779)

Co-authored-by: Łukasz Langa <[email protected]>

Author: aorcajo

Lib/_pyrepl/readline.py

@@ -427,12 +427,16 @@ def read_history_file(self, filename: str = gethistoryfile()) -> None:
         history = self.get_reader().history
 
         with open(os.path.expanduser(filename), 'rb') as f:
-            lines = [line.decode('utf-8', errors='replace') for line in f.read().split(b'\n')]
+            is_editline = f.readline().startswith(b"_HiStOrY_V2_")
+            if is_editline:
+                encoding = "unicode-escape"
+            else:
+                f.seek(0)
+                encoding = "utf-8"
+
+            lines = [line.decode(encoding, errors='replace') for line in f.read().split(b'\n')]
             buffer = []
             for line in lines:
-                # Ignore readline history file header
-                if line.startswith("_HiStOrY_V2_"):
-                    continue
                 if line.endswith("\r"):
                     buffer.append(line+'\n')
                 else:

Lib/test/test_pyrepl/test_pyrepl.py

@@ -1216,12 +1216,34 @@ def run_repl(
         *,
         cmdline_args: list[str] | None = None,
         cwd: str | None = None,
+    ) -> tuple[str, int]:
+        temp_dir = None
+        if cwd is None:
+            temp_dir = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
+            cwd = temp_dir.name
+        try:
+            return self._run_repl(
+                repl_input, env=env, cmdline_args=cmdline_args, cwd=cwd
+            )
+        finally:
+            if temp_dir is not None:
+                temp_dir.cleanup()
+
+    def _run_repl(
+        self,
+        repl_input: str | list[str],
+        *,
+        env: dict | None,
+        cmdline_args: list[str] | None,
+        cwd: str,
     ) -> tuple[str, int]:
         assert pty
         master_fd, slave_fd = pty.openpty()
         cmd = [sys.executable, "-i", "-u"]
         if env is None:
             cmd.append("-I")
+        elif "PYTHON_HISTORY" not in env:
+            env["PYTHON_HISTORY"] = os.path.join(cwd, ".regrtest_history")
         if cmdline_args is not None:
             cmd.extend(cmdline_args)
         process = subprocess.Popen(
@@ -1260,3 +1282,26 @@ def run_repl(
             process.kill()
             exit_code = process.wait()
         return "".join(output), exit_code
+
+    def test_readline_history_file(self):
+        # skip, if readline module is not available
+        readline = import_module('readline')
+        if readline.backend != "editline":
+            self.skipTest("GNU readline is not affected by this issue")
+
+        hfile = tempfile.NamedTemporaryFile()
+        self.addCleanup(unlink, hfile.name)
+        env = os.environ.copy()
+        env["PYTHON_HISTORY"] = hfile.name
+
+        env["PYTHON_BASIC_REPL"] = "1"
+        output, exit_code = self.run_repl("spam \nexit()\n", env=env)
+        self.assertEqual(exit_code, 0)
+        self.assertIn("spam ", output)
+        self.assertNotEqual(pathlib.Path(hfile.name).stat().st_size, 0)
+        self.assertIn("spam\\040", pathlib.Path(hfile.name).read_text())
+
+        env.pop("PYTHON_BASIC_REPL", None)
+        output, exit_code = self.run_repl("exit\n", env=env)
+        self.assertEqual(exit_code, 0)
+        self.assertNotIn("\\040", pathlib.Path(hfile.name).read_text())

Lib/test/test_repl.py

@@ -41,7 +41,7 @@ def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw):
     # path may be used by Py_GetPath() to build the default module search
     # path.
     stdin_fname = os.path.join(os.path.dirname(sys.executable), "<stdin>")
-    cmd_line = [stdin_fname, '-E', '-i']
+    cmd_line = [stdin_fname, '-I', '-i']
     cmd_line.extend(args)
 
     # Set TERM=vt100, for the rationale see the comments in spawn_python() of
@@ -228,6 +228,7 @@ def test_asyncio_repl_reaches_python_startup_script(self):
                 f.write("exit(0)" + os.linesep)
 
             env = os.environ.copy()
+            env["PYTHON_HISTORY"] = os.path.join(tmpdir, ".asyncio_history")
             env["PYTHONSTARTUP"] = script
             subprocess.check_call(
                 [sys.executable, "-m", "asyncio"],
@@ -240,7 +241,7 @@ def test_asyncio_repl_reaches_python_startup_script(self):
     @unittest.skipUnless(pty, "requires pty")
     def test_asyncio_repl_is_ok(self):
         m, s = pty.openpty()
-        cmd = [sys.executable, "-m", "asyncio"]
+        cmd = [sys.executable, "-I", "-m", "asyncio"]
         proc = subprocess.Popen(
             cmd,
             stdin=s,

Misc/NEWS.d/next/Core_and_Builtins/2024-09-06-14-13-01.gh-issue-119310.WQxyDF.rst

@@ -0,0 +1,3 @@
+Allow the new interactive shell to read history files written with the
+editline library that use unicode-escaped entries.  Patch by aorcajo and
+Łukasz Langa.