bpo-38470: Fix test_compileall.test_compile_dir_maxlevels() (GH-16789)

Fix test_compile_dir_maxlevels() on Windows without long path
support: only create 3 subdirectories instead of between 20 and 100
subdirectories.

Fix also compile_dir() to use the current sys.getrecursionlimit()
value as the default maxlevels value, rather than using
sys.getrecursionlimit() value read at startup.

Author: vstinner

Lib/compileall.py

@@ -19,11 +19,9 @@
 from functools import partial
 from pathlib import Path
 
-RECURSION_LIMIT = sys.getrecursionlimit()
-
 __all__ = ["compile_dir","compile_file","compile_path"]
 
-def _walk_dir(dir, maxlevels=RECURSION_LIMIT, quiet=0):
+def _walk_dir(dir, maxlevels, quiet=0):
     if quiet < 2 and isinstance(dir, os.PathLike):
         dir = os.fspath(dir)
     if not quiet:
@@ -46,7 +44,7 @@ def _walk_dir(dir, maxlevels=RECURSION_LIMIT, quiet=0):
             yield from _walk_dir(fullname, maxlevels=maxlevels - 1,
                                  quiet=quiet)
 
-def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False,
+def compile_dir(dir, maxlevels=None, ddir=None, force=False,
                 rx=None, quiet=0, legacy=False, optimize=-1, workers=1,
                 invalidation_mode=None, stripdir=None,
                 prependdir=None, limit_sl_dest=None):
@@ -83,6 +81,8 @@ def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False,
             from concurrent.futures import ProcessPoolExecutor
         except ImportError:
             workers = 1
+    if maxlevels is None:
+        maxlevels = sys.getrecursionlimit()
     files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels)
     success = True
     if workers != 1 and ProcessPoolExecutor is not None:
@@ -285,7 +285,7 @@ def main():
     parser = argparse.ArgumentParser(
         description='Utilities to support installing Python libraries.')
     parser.add_argument('-l', action='store_const', const=0,
-                        default=RECURSION_LIMIT, dest='maxlevels',
+                        default=None, dest='maxlevels',
                         help="don't recurse into subdirectories")
     parser.add_argument('-r', type=int, dest='recursion',
                         help=('control the maximum recursion level. '

Lib/test/test_compileall.py

@@ -46,57 +46,6 @@ def setUp(self):
     def tearDown(self):
         shutil.rmtree(self.directory)
 
-    def create_long_path(self):
-        long_path = os.path.join(self.directory, "long")
-
-        # Create a long path, 10 directories at a time.
-        # It will be 100 directories deep, or shorter if the OS limits it.
-        for i in range(10):
-            longer_path = os.path.join(
-                long_path, *(f"dir_{i}_{j}" for j in range(10))
-            )
-
-            # Check if we can open __pycache__/*.pyc.
-            # Also, put in the source file that we want to compile
-            longer_source = os.path.join(longer_path, '_test_long.py')
-            longer_cache = importlib.util.cache_from_source(longer_source)
-            try:
-                os.makedirs(longer_path)
-                shutil.copyfile(self.source_path, longer_source)
-                os.makedirs(os.path.dirname(longer_cache))
-                # Make sure we can write to the cache
-                with open(longer_cache, 'w'):
-                    pass
-            except FileNotFoundError:
-                # On Windows, a  FileNotFoundError("The filename or extension
-                # is too long") is raised for long paths
-                if sys.platform == "win32":
-                    break
-                else:
-                    raise
-            except OSError as exc:
-                if exc.errno == errno.ENAMETOOLONG:
-                    break
-                else:
-                    raise
-
-            # Remove the __pycache__
-            shutil.rmtree(os.path.dirname(longer_cache))
-
-            long_path = longer_path
-            long_source = longer_source
-            long_cache = longer_cache
-
-        # On Windows, MAX_PATH is 260 characters, our path with the 20
-        # directories is 160 characters long, leaving something for the
-        # root (self.directory) as well.
-        # Tests assume long_path contains at least 10 directories.
-        if i < 2:
-            raise ValueError(f'"Long path" is too short: {long_path}')
-
-        self.source_path_long = long_source
-        self.bc_path_long = long_cache
-
     def add_bad_source_file(self):
         self.bad_source_path = os.path.join(self.directory, '_test_bad.py')
         with open(self.bad_source_path, 'w') as file:
@@ -247,14 +196,21 @@ def test_compile_missing_multiprocessing(self, compile_file_mock):
         self.assertTrue(compile_file_mock.called)
 
     def test_compile_dir_maxlevels(self):
-        # Test the actual impact of maxlevels attr
-        self.create_long_path()
-        compileall.compile_dir(os.path.join(self.directory, "long"),
-                               maxlevels=10, quiet=True)
-        self.assertFalse(os.path.isfile(self.bc_path_long))
-        compileall.compile_dir(os.path.join(self.directory, "long"),
-                               quiet=True)
-        self.assertTrue(os.path.isfile(self.bc_path_long))
+        # Test the actual impact of maxlevels parameter
+        depth = 3
+        path = self.directory
+        for i in range(1, depth + 1):
+            path = os.path.join(path, f"dir_{i}")
+            source = os.path.join(path, 'script.py')
+            os.mkdir(path)
+            shutil.copyfile(self.source_path, source)
+        pyc_filename = importlib.util.cache_from_source(source)
+
+        compileall.compile_dir(self.directory, quiet=True, maxlevels=depth - 1)
+        self.assertFalse(os.path.isfile(pyc_filename))
+
+        compileall.compile_dir(self.directory, quiet=True, maxlevels=depth)
+        self.assertTrue(os.path.isfile(pyc_filename))
 
     def test_strip_only(self):
         fullpath = ["test", "build", "real", "path"]

Misc/NEWS.d/next/Tests/2019-10-14-22-46-35.bpo-38470.NHtzpy.rst

@@ -0,0 +1,3 @@
+Fix ``test_compileall.test_compile_dir_maxlevels()`` on Windows without long
+path support: only create 3 subdirectories instead of between 20 and 100
+subdirectories.