gh-100238: Use setuptools in peg-generator and reenable tests (#104798)

Author: lysnikolaou

Lib/test/support/__init__.py

@@ -1865,15 +1865,16 @@ def missing_compiler_executable(cmd_names=[]):
     missing.
 
     """
-    # TODO (PEP 632): alternate check without using distutils
-    from distutils import ccompiler, sysconfig, spawn, errors
+    from setuptools._distutils import ccompiler, sysconfig, spawn
+    from setuptools import errors
+
     compiler = ccompiler.new_compiler()
     sysconfig.customize_compiler(compiler)
     if compiler.compiler_type == "msvc":
         # MSVC has no executables, so check whether initialization succeeds
         try:
             compiler.initialize()
-        except errors.DistutilsPlatformError:
+        except errors.PlatformError:
             return "msvc"
     for name in compiler.executables:
         if cmd_names and name not in cmd_names:
@@ -2270,6 +2271,42 @@ def requires_venv_with_pip():
     return unittest.skipUnless(ctypes, 'venv: pip requires ctypes')
 
 
+# Context manager that creates a virtual environment, install setuptools and wheel in it
+# and returns the path to the venv directory and the path to the python executable
+@contextlib.contextmanager
+def setup_venv_with_pip_setuptools_wheel(venv_dir):
+    import subprocess
+    from .os_helper import temp_cwd
+
+    with temp_cwd() as temp_dir:
+        # Create virtual environment to get setuptools
+        cmd = [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir]
+        if verbose:
+            print()
+            print('Run:', ' '.join(cmd))
+        subprocess.run(cmd, check=True)
+
+        venv = os.path.join(temp_dir, venv_dir)
+
+        # Get the Python executable of the venv
+        python_exe = os.path.basename(sys.executable)
+        if sys.platform == 'win32':
+            python = os.path.join(venv, 'Scripts', python_exe)
+        else:
+            python = os.path.join(venv, 'bin', python_exe)
+
+        cmd = [python, '-X', 'dev',
+               '-m', 'pip', 'install',
+               findfile('setuptools-67.6.1-py3-none-any.whl'),
+               findfile('wheel-0.40.0-py3-none-any.whl')]
+        if verbose:
+            print()
+            print('Run:', ' '.join(cmd))
+        subprocess.run(cmd, check=True)
+
+        yield python
+
+
 # True if Python is built with the Py_DEBUG macro defined: if
 # Python is built in debug mode (./configure --with-pydebug).
 Py_DEBUG = hasattr(sys, 'gettotalrefcount')

Lib/test/test_cppext.py

@@ -35,39 +35,20 @@ def test_build_cpp03(self):
     # the test uses venv+pip: skip if it's not available
     @support.requires_venv_with_pip()
     def check_build(self, std_cpp03, extension_name):
-        # Build in a temporary directory
-        with os_helper.temp_cwd():
-            self._check_build(std_cpp03, extension_name)
+        venv_dir = 'env'
+        with support.setup_venv_with_pip_setuptools_wheel(venv_dir) as python_exe:
+            self._check_build(std_cpp03, extension_name, python_exe)
 
-    def _check_build(self, std_cpp03, extension_name):
+    def _check_build(self, std_cpp03, extension_name, python_exe):
         pkg_dir = 'pkg'
         os.mkdir(pkg_dir)
         shutil.copy(SETUP_TESTCPPEXT, os.path.join(pkg_dir, "setup.py"))
 
-        venv_dir = 'env'
-        verbose = support.verbose
-
-        # Create virtual environment to get setuptools
-        cmd = [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir]
-        if verbose:
-            print()
-            print('Run:', ' '.join(cmd))
-        subprocess.run(cmd, check=True)
-
-        # Get the Python executable of the venv
-        python_exe = 'python'
-        if sys.executable.endswith('.exe'):
-            python_exe += '.exe'
-        if MS_WINDOWS:
-            python = os.path.join(venv_dir, 'Scripts', python_exe)
-        else:
-            python = os.path.join(venv_dir, 'bin', python_exe)
-
         def run_cmd(operation, cmd):
             env = os.environ.copy()
             env['CPYTHON_TEST_CPP_STD'] = 'c++03' if std_cpp03 else 'c++11'
             env['CPYTHON_TEST_EXT_NAME'] = extension_name
-            if verbose:
+            if support.verbose:
                 print('Run:', ' '.join(cmd))
                 subprocess.run(cmd, check=True, env=env)
             else:
@@ -81,29 +62,23 @@ def run_cmd(operation, cmd):
                     self.fail(
                         f"{operation} failed with exit code {proc.returncode}")
 
-        cmd = [python, '-X', 'dev',
-               '-m', 'pip', 'install',
-               support.findfile('setuptools-67.6.1-py3-none-any.whl'),
-               support.findfile('wheel-0.40.0-py3-none-any.whl')]
-        run_cmd('Install build dependencies', cmd)
-
         # Build and install the C++ extension
-        cmd = [python, '-X', 'dev',
+        cmd = [python_exe, '-X', 'dev',
                '-m', 'pip', 'install', '--no-build-isolation',
                os.path.abspath(pkg_dir)]
         run_cmd('Install', cmd)
 
         # Do a reference run. Until we test that running python
         # doesn't leak references (gh-94755), run it so one can manually check
         # -X showrefcount results against this baseline.
-        cmd = [python,
+        cmd = [python_exe,
                '-X', 'dev',
                '-X', 'showrefcount',
                '-c', 'pass']
         run_cmd('Reference run', cmd)
 
         # Import the C++ extension
-        cmd = [python,
+        cmd = [python_exe,
                '-X', 'dev',
                '-X', 'showrefcount',
                '-c', f"import {extension_name}"]

Lib/test/test_peg_generator/__init__.py

@@ -3,9 +3,6 @@
 from test import support
 from test.support import load_package_tests
 
-# TODO: gh-92584: peg_generator uses distutils which was removed in Python 3.12
-raise unittest.SkipTest("distutils has been removed in Python 3.12")
-
 
 if support.check_sanitizer(address=True, memory=True):
     # bpo-46633: Skip the test because it is too slow when Python is built

Lib/test/test_peg_generator/test_c_parser.py

@@ -1,3 +1,5 @@
+import contextlib
+import subprocess
 import sysconfig
 import textwrap
 import unittest
@@ -8,7 +10,7 @@
 
 from test import test_tools
 from test import support
-from test.support import os_helper
+from test.support import os_helper, import_helper
 from test.support.script_helper import assert_python_ok
 
 _py_cflags_nodist = sysconfig.get_config_var("PY_CFLAGS_NODIST")
@@ -88,6 +90,16 @@ def setUpClass(cls):
         cls.library_dir = tempfile.mkdtemp(dir=cls.tmp_base)
         cls.addClassCleanup(shutil.rmtree, cls.library_dir)
 
+        with contextlib.ExitStack() as stack:
+            python_exe = stack.enter_context(support.setup_venv_with_pip_setuptools_wheel("venv"))
+            sitepackages = subprocess.check_output(
+                [python_exe, "-c", "import sysconfig; print(sysconfig.get_path('platlib'))"],
+                text=True,
+            ).strip()
+            stack.enter_context(import_helper.DirsOnSysPath(sitepackages))
+            cls.addClassCleanup(stack.pop_all().close)
+
+    @support.requires_venv_with_pip()
     def setUp(self):
         self._backup_config_vars = dict(sysconfig._CONFIG_VARS)
         cmd = support.missing_compiler_executable()