changeset:   96716:cb9df1ae287b
branch:      2.7
user:        Jason R. Coombs <jaraco@jaraco.com>
date:        Sun Jun 28 13:05:19 2015 -0400
files:       Lib/tokenize.py Misc/NEWS
description:
Issue #20387: Backport fix from Python 3.4


diff -r 524a0e755797 -r cb9df1ae287b Lib/tokenize.py
--- a/Lib/tokenize.py	Sun Jun 28 13:03:26 2015 -0400
+++ b/Lib/tokenize.py	Sun Jun 28 13:05:19 2015 -0400
@@ -198,6 +198,8 @@
 
     def untokenize(self, iterable):
         it = iter(iterable)
+        indents = []
+        startline = False
         for t in it:
             if len(t) == 2:
                 self.compat(t, it)
@@ -205,6 +207,21 @@
             tok_type, token, start, end, line = t
             if tok_type == ENDMARKER:
                 break
+            if tok_type == INDENT:
+                indents.append(token)
+                continue
+            elif tok_type == DEDENT:
+                indents.pop()
+                self.prev_row, self.prev_col = end
+                continue
+            elif tok_type in (NEWLINE, NL):
+                startline = True
+            elif startline and indents:
+                indent = indents[-1]
+                if start[1] >= len(indent):
+                    self.tokens.append(indent)
+                    self.prev_col = len(indent)
+                startline = False
             self.add_whitespace(start)
             self.tokens.append(token)
             self.prev_row, self.prev_col = end
diff -r 524a0e755797 -r cb9df1ae287b Misc/NEWS
--- a/Misc/NEWS	Sun Jun 28 13:03:26 2015 -0400
+++ b/Misc/NEWS	Sun Jun 28 13:05:19 2015 -0400
@@ -30,6 +30,9 @@
 Library
 -------
 
+- Issue #20387: Restore semantic round-trip correctness in tokenize/untokenize
+  for tab-indented blocks.
+
 - Issue #24456: Fixed possible buffer over-read in adpcm2lin() and lin2adpcm()
   functions of the audioop module.  Fixed SystemError when the state is not a
   tuple.  Fixed possible memory leak.