changeset:   94922:f805fdacdfe0
user:        Victor Stinner <victor.stinner@gmail.com>
date:        Tue Mar 10 13:20:34 2015 +0100
files:       Doc/library/os.rst Lib/os.py Misc/NEWS
description:
Issue #23605: os.walk() now calls os.scandir() instead of os.listdir().
The usage of os.scandir() reduces the number of calls to os.stat().
Initial patch written by Ben Hoyt.


diff -r 52c7017fdcdd -r f805fdacdfe0 Doc/library/os.rst
--- a/Doc/library/os.rst	Mon Mar 09 15:55:37 2015 +0100
+++ b/Doc/library/os.rst	Tue Mar 10 13:20:34 2015 +0100
@@ -2618,6 +2618,11 @@
           for name in dirs:
               os.rmdir(os.path.join(root, name))
 
+   .. versionchanged:: 3.5
+      The function now calls :func:`os.scandir` instead of :func:`os.listdir`.
+      The usage of :func:`os.scandir` reduces the number of calls to
+      :func:`os.stat`.
+
 
 .. function:: fwalk(top='.', topdown=True, onerror=None, *, follow_symlinks=False, dir_fd=None)
 
diff -r 52c7017fdcdd -r f805fdacdfe0 Lib/os.py
--- a/Lib/os.py	Mon Mar 09 15:55:37 2015 +0100
+++ b/Lib/os.py	Tue Mar 10 13:20:34 2015 +0100
@@ -323,7 +323,7 @@
     the value of topdown, the list of subdirectories is retrieved before the
     tuples for the directory and its subdirectories are generated.
 
-    By default errors from the os.listdir() call are ignored.  If
+    By default errors from the os.scandir() call are ignored.  If
     optional arg 'onerror' is specified, it should be a function; it
     will be called with one argument, an OSError instance.  It can
     report the error to continue with the walk, or raise the exception
@@ -352,7 +352,9 @@
 
     """
 
-    islink, join, isdir = path.islink, path.join, path.isdir
+    dirs = []
+    nondirs = []
+    symlinks = set()
 
     # We may not have read permission for top, in which case we can't
     # get a list of the files the directory contains.  os.walk
@@ -360,27 +362,46 @@
     # minor reason when (say) a thousand readable directories are still
     # left to visit.  That logic is copied here.
     try:
-        # Note that listdir is global in this module due
+        # Note that scandir is global in this module due
         # to earlier import-*.
-        names = listdir(top)
-    except OSError as err:
+        for entry in scandir(top):
+            try:
+                is_dir = entry.is_dir()
+            except OSError:
+                # If is_dir() raises an OSError, consider that the entry is not
+                # a directory, same behaviour than os.path.isdir().
+                is_dir = False
+
+            if is_dir:
+                dirs.append(entry.name)
+
+                try:
+                    if entry.is_symlink():
+                        symlinks.add(entry.name)
+                except OSError:
+                    # If is_symlink() raises an OSError, consider that the
+                    # entry is not a symbolik link, same behaviour than
+                    # os.path.islink().
+                    pass
+            else:
+                nondirs.append(entry.name)
+    except OSError as error:
+        # scandir() or iterating into scandir() iterator raised an OSError
         if onerror is not None:
-            onerror(err)
+            onerror(error)
         return
 
-    dirs, nondirs = [], []
-    for name in names:
-        if isdir(join(top, name)):
-            dirs.append(name)
-        else:
-            nondirs.append(name)
-
+    # Yield before recursion if going top down
     if topdown:
         yield top, dirs, nondirs
+
+    # Recurse into sub-directories
     for name in dirs:
-        new_path = join(top, name)
-        if followlinks or not islink(new_path):
+        if followlinks or name not in symlinks:
+            new_path = path.join(top, name)
             yield from walk(new_path, topdown, onerror, followlinks)
+
+    # Yield after recursion if going bottom up
     if not topdown:
         yield top, dirs, nondirs
 
diff -r 52c7017fdcdd -r f805fdacdfe0 Misc/NEWS
--- a/Misc/NEWS	Mon Mar 09 15:55:37 2015 +0100
+++ b/Misc/NEWS	Tue Mar 10 13:20:34 2015 +0100
@@ -13,6 +13,10 @@
 Library
 -------
 
+- Issue #23605: os.walk() now calls os.scandir() instead of os.listdir().
+  The usage of os.scandir() reduces the number of calls to os.stat().
+  Initial patch written by Ben Hoyt.
+
 
 What's New in Python 3.5 alpha 2?
 =================================