changeset:   96715:524a0e755797
branch:      2.7
parent:      96702:1f6c096ee772
user:        Jason R. Coombs <jaraco@jaraco.com>
date:        Sun Jun 28 13:03:26 2015 -0400
files:       Lib/test/test_tokenize.py
description:
Issue #20387: Backport test from Python 3.4


diff -r 1f6c096ee772 -r 524a0e755797 Lib/test/test_tokenize.py
--- a/Lib/test/test_tokenize.py	Sun Jun 28 17:51:40 2015 +0300
+++ b/Lib/test/test_tokenize.py	Sun Jun 28 13:03:26 2015 -0400
@@ -559,7 +559,7 @@
 
 from test import test_support
 from tokenize import (untokenize, generate_tokens, NUMBER, NAME, OP,
-                     STRING, ENDMARKER, tok_name, Untokenizer)
+                     STRING, ENDMARKER, tok_name, Untokenizer, tokenize)
 from StringIO import StringIO
 import os
 from unittest import TestCase
@@ -650,12 +650,30 @@
         self.assertEqual(u.untokenize(iter([token])), 'Hello ')
 
 
+class TestRoundtrip(TestCase):
+    def roundtrip(self, code):
+        if isinstance(code, str):
+            code = code.encode('utf-8')
+        tokens = generate_tokens(StringIO(code).readline)
+        return untokenize(tokens).decode('utf-8')
+
+    def test_indentation_semantics_retained(self):
+        """
+        Ensure that although whitespace might be mutated in a roundtrip,
+        the semantic meaning of the indentation remains consistent.
+        """
+        code = "if False:\n\tx=3\n\tx=3\n"
+        codelines = self.roundtrip(code).split('\n')
+        self.assertEqual(codelines[1], codelines[2])
+
+
 __test__ = {"doctests" : doctests, 'decistmt': decistmt}
 
 def test_main():
     from test import test_tokenize
     test_support.run_doctest(test_tokenize, True)
     test_support.run_unittest(UntokenizeTest)
+    test_support.run_unittest(TestRoundtrip)
 
 if __name__ == "__main__":
     test_main()