Fix the fix of the fixer fix

Author: vstinner

Lib/test/libregrtest/run_workers.py

@@ -17,10 +17,10 @@
 from .logger import Logger
 from .result import TestResult, State
 from .results import TestResults
-from .runtests import RunTests, JsonFileType, JSON_FILE_USE_FILENAME
+from .runtests import RunTests, JsonFileType
 from .single import PROGRESS_MIN_TIME
 from .utils import (
-    StrPath, StrJSON, TestName, MS_WINDOWS,
+    StrPath, StrJSON, TestName, MS_WINDOWS, TMP_PREFIX,
     format_duration, print_warning, count, plural)
 from .worker import create_worker_process, USE_PROCESS_GROUP
 
@@ -228,9 +228,14 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult:
         err_msg = None
 
         stdout_file = tempfile.TemporaryFile('w+', encoding=encoding)
-        if JSON_FILE_USE_FILENAME:
-            json_tmpfile = tempfile.NamedTemporaryFile('w+', encoding='utf8')
-            print("main process: create NamedTemporaryFile")
+
+        json_file_use_filename = self.runtests.json_file_use_filename()
+        print("main process: json_file_use_filename?", json_file_use_filename)
+        if json_file_use_filename:
+            prefix = TMP_PREFIX + 'json_'
+            json_tmpfile = tempfile.NamedTemporaryFile('w+', encoding='utf8',
+                                                       prefix=prefix)
+            print("main process: create NamedTemporaryFile", json_tmpfile.name)
         else:
             json_tmpfile = tempfile.TemporaryFile('w+', encoding='utf8')
             print("main process: create TemporaryFile")
@@ -239,14 +244,14 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult:
         # non-blocking pipes on Emscripten with NodeJS.
         with (stdout_file, json_tmpfile):
             stdout_fd = stdout_file.fileno()
-            if JSON_FILE_USE_FILENAME:
+            if json_file_use_filename:
                 json_file = json_tmpfile.name
             else:
                 json_file = json_tmpfile.fileno()
                 if MS_WINDOWS:
                     json_file = msvcrt.get_osfhandle(json_file)
-            print("main process json_type file:", type(json_file))
-            print("main process json_type:", json_file)
+            print("main process json_file type:", type(json_file))
+            print("main process json_file:", json_file)
 
             kwargs = {}
             if match_tests:

Lib/test/libregrtest/runtests.py

@@ -8,20 +8,10 @@
     StrPath, StrJSON, TestTuple, FilterTuple, FilterDict)
 
 
-if support.is_emscripten or support.is_wasi:
-    # On Emscripten/WASI, it's a filename. Passing a file descriptor to a
-    # worker process fails with "OSError: [Errno 8] Bad file descriptor" in the
-    # worker process.
-    JsonFileType = StrPath
-    JSON_FILE_USE_FILENAME = True
-else:
-    # On Unix, it's a file descriptor.
-    # On Windows, it's a handle.
-    JsonFileType = int
-    JSON_FILE_USE_FILENAME = False
+# See RunTests.json_file_use_filename()
+JsonFileType = int | StrPath
 
 import os
-print(os.getpid(), "JSON_FILE_USE_FILENAME:", JSON_FILE_USE_FILENAME)
 print(os.getpid(), "JsonFileType:", JsonFileType)
 
 
@@ -58,7 +48,7 @@ class RunTests:
     python_cmd: tuple[str] | None
     randomize: bool
     random_seed: int | None
-    json_file: JsonFileType  | None
+    json_file: JsonFileType | None
 
     def copy(self, **override):
         state = dataclasses.asdict(self)
@@ -92,6 +82,18 @@ def as_json(self) -> StrJSON:
     def from_json(worker_json: StrJSON) -> 'RunTests':
         return json.loads(worker_json, object_hook=_decode_runtests)
 
+    def json_file_use_filename(self):
+        # On Unix, it's a file descriptor.
+        # On Windows, it's a handle.
+        # On Emscripten/WASI, it's a filename. Passing a file descriptor to a
+        # worker process fails with "OSError: [Errno 8] Bad file descriptor" in the
+        # worker process.
+        return (
+            self.python_cmd
+            or support.is_emscripten
+            or support.is_wasi
+        )
+
 
 class _EncodeRunTests(json.JSONEncoder):
     def default(self, o: Any) -> dict[str, Any]:

Lib/test/libregrtest/utils.py

@@ -17,8 +17,12 @@
 
 
 MS_WINDOWS = (sys.platform == 'win32')
-WORK_DIR_PREFIX = 'test_python_'
-WORKER_WORK_DIR_PREFIX = f'{WORK_DIR_PREFIX}worker_'
+
+# All temporary files and temporary directories created by libregrtest should
+# use TMP_PREFIX so cleanup_temp_dir() can remove them all.
+TMP_PREFIX = 'test_python_'
+WORK_DIR_PREFIX = TMP_PREFIX
+WORKER_WORK_DIR_PREFIX = WORK_DIR_PREFIX + 'worker_'
 
 # bpo-38203: Maximum delay in seconds to exit Python (call Py_Finalize()).
 # Used to protect against threading._shutdown() hang.
@@ -580,7 +584,7 @@ def display_header():
 def cleanup_temp_dir(tmp_dir: StrPath):
     import glob
 
-    path = os.path.join(glob.escape(tmp_dir), WORK_DIR_PREFIX + '*')
+    path = os.path.join(glob.escape(tmp_dir), TMP_PREFIX + '*')
     print("Cleanup %s directory" % tmp_dir)
     for name in glob.glob(path):
         if os.path.isdir(name):

Lib/test/libregrtest/worker.py

@@ -7,7 +7,7 @@
 from test.support import os_helper
 
 from .setup import setup_process, setup_test_dir
-from .runtests import RunTests, JsonFileType, JSON_FILE_USE_FILENAME
+from .runtests import RunTests, JsonFileType
 from .single import run_single_test
 from .utils import (
     StrPath, StrJSON, FilterTuple, MS_WINDOWS,
@@ -57,17 +57,23 @@ def create_worker_process(runtests: RunTests,
         close_fds=True,
         cwd=work_dir,
     )
-    if JSON_FILE_USE_FILENAME:
-        # Nothing to do to pass the JSON filename to the worker process
+
+    # Pass json_file to the worker process
+    if isinstance(json_file, str):
+        # Filename: nothing to do to
+        print("create_worker_process() json_file: filename")
         pass
     elif MS_WINDOWS:
-        # Pass the JSON handle to the worker process
+        # Windows handle
+        print("create_worker_process() json_file: Windows handle")
         startupinfo = subprocess.STARTUPINFO()
         startupinfo.lpAttributeList = {"handle_list": [json_file]}
         kwargs['startupinfo'] = startupinfo
     else:
-        # Pass the JSON file descriptor to the worker process
+        # Unix file descriptor
+        print("create_worker_process() json_file: Unix fd")
         kwargs['pass_fds'] = [json_file]
+
     if USE_PROCESS_GROUP:
         kwargs['start_new_session'] = True