bpo-40334: Add support for feature_version in new PEG parser (GH-19827)

`ast.parse` and `compile` support a `feature_version` parameter that
tells the parser to parse the input string, as if it were written in
an older Python version.
The `feature_version` is propagated to the tokenizer, which uses it
to handle the three different stages of support for `async` and
`await`. Additionally, it disallows the following at parser level:
- The '@' operator in < 3.5
- Async functions in < 3.5
- Async comprehensions in < 3.6
- Underscores in numeric literals in < 3.6
- Await expression in < 3.5
- Variable annotations in < 3.6
- Async for-loops in < 3.5
- Async with-statements in < 3.5
- F-strings in < 3.6

Closes we-like-parsers#124.

Author: lysnikolaou

Grammar/python.gram

@@ -80,30 +80,34 @@ compound_stmt[stmt_ty]:
 # NOTE: annotated_rhs may start with 'yield'; yield_expr must start with 'yield'
 assignment:
     | a=NAME ':' b=expression c=['=' d=annotated_rhs { d }] {
-        _Py_AnnAssign(CHECK(_PyPegen_set_expr_context(p, a, Store)), b, c, 1, EXTRA) }
+        CHECK_VERSION(
+            6,
+            "Variable annotation syntax is",
+            _Py_AnnAssign(CHECK(_PyPegen_set_expr_context(p, a, Store)), b, c, 1, EXTRA)
+        ) }
     | a=('(' b=inside_paren_ann_assign_target ')' { b }
          | ann_assign_subscript_attribute_target) ':' b=expression c=['=' d=annotated_rhs { d }] {
-        _Py_AnnAssign(a, b, c, 0, EXTRA)}
+        CHECK_VERSION(6, "Variable annotations syntax is", _Py_AnnAssign(a, b, c, 0, EXTRA)) }
     | a=(z=star_targets '=' { z })+ b=(yield_expr | star_expressions) tc=[TYPE_COMMENT] {
          _Py_Assign(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) }
     | a=target b=augassign c=(yield_expr | star_expressions) {
          _Py_AugAssign(a, b->kind, c, EXTRA) }
     | invalid_assignment
 
 augassign[AugOperator*]:
-    | '+=' {_PyPegen_augoperator(p, Add)}
-    | '-=' {_PyPegen_augoperator(p, Sub)}
-    | '*=' {_PyPegen_augoperator(p, Mult)}
-    | '@=' {_PyPegen_augoperator(p, MatMult)}
-    | '/=' {_PyPegen_augoperator(p, Div)}
-    | '%=' {_PyPegen_augoperator(p, Mod)}
-    | '&=' {_PyPegen_augoperator(p, BitAnd)}
-    | '|=' {_PyPegen_augoperator(p, BitOr)}
-    | '^=' {_PyPegen_augoperator(p, BitXor)}
-    | '<<=' {_PyPegen_augoperator(p, LShift)}
-    | '>>=' {_PyPegen_augoperator(p, RShift)}
-    | '**=' {_PyPegen_augoperator(p, Pow)}
-    | '//=' {_PyPegen_augoperator(p, FloorDiv)}
+    | '+=' { _PyPegen_augoperator(p, Add) }
+    | '-=' { _PyPegen_augoperator(p, Sub) }
+    | '*=' { _PyPegen_augoperator(p, Mult) }
+    | '@=' { CHECK_VERSION(5, "The '@' operator is", _PyPegen_augoperator(p, MatMult)) }
+    | '/=' { _PyPegen_augoperator(p, Div) }
+    | '%=' { _PyPegen_augoperator(p, Mod) }
+    | '&=' { _PyPegen_augoperator(p, BitAnd) }
+    | '|=' { _PyPegen_augoperator(p, BitOr) }
+    | '^=' { _PyPegen_augoperator(p, BitXor) }
+    | '<<=' { _PyPegen_augoperator(p, LShift) }
+    | '>>=' { _PyPegen_augoperator(p, RShift) }
+    | '**=' { _PyPegen_augoperator(p, Pow) }
+    | '//=' { _PyPegen_augoperator(p, FloorDiv) }
 
 global_stmt[stmt_ty]: 'global' a=','.NAME+ {
     _Py_Global(CHECK(_PyPegen_map_names_to_ids(p, a)), EXTRA) }
@@ -156,14 +160,20 @@ while_stmt[stmt_ty]:
     | 'while' a=named_expression ':' b=block c=[else_block] { _Py_While(a, b, c, EXTRA) }
 
 for_stmt[stmt_ty]:
-    | is_async=[ASYNC] 'for' t=star_targets 'in' ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] {
-        (is_async ? _Py_AsyncFor : _Py_For)(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA) }
+    | 'for' t=star_targets 'in' ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] {
+        _Py_For(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA) }
+    | ASYNC 'for' t=star_targets 'in' ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] {
+        CHECK_VERSION(5, "Async for loops are", _Py_AsyncFor(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA)) }
 
 with_stmt[stmt_ty]:
-    | is_async=[ASYNC] 'with' '(' a=','.with_item+ ')' ':' b=block {
-        (is_async ? _Py_AsyncWith : _Py_With)(a, b, NULL, EXTRA) }
-    | is_async=[ASYNC] 'with' a=','.with_item+ ':' tc=[TYPE_COMMENT] b=block {
-        (is_async ? _Py_AsyncWith : _Py_With)(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) }
+    | 'with' '(' a=','.with_item+ ')' ':' b=block {
+        _Py_With(a, b, NULL, EXTRA) }
+    | 'with' a=','.with_item+ ':' tc=[TYPE_COMMENT] b=block {
+        _Py_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) }
+    | ASYNC 'with' '(' a=','.with_item+ ')' ':' b=block {
+       CHECK_VERSION(5, "Async with statements are", _Py_AsyncWith(a, b, NULL, EXTRA)) }
+    | ASYNC 'with' a=','.with_item+ ':' tc=[TYPE_COMMENT] b=block {
+       CHECK_VERSION(5, "Async with statements are", _Py_AsyncWith(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA)) }
 with_item[withitem_ty]:
     | e=expression o=['as' t=target { t }] { _Py_withitem(e, o, p->arena) }
 
@@ -188,10 +198,18 @@ function_def[stmt_ty]:
     | function_def_raw
 
 function_def_raw[stmt_ty]:
-    | is_async=[ASYNC] 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block {
-        (is_async ? _Py_AsyncFunctionDef : _Py_FunctionDef)(n->v.Name.id,
-                             (params) ? params : CHECK(_PyPegen_empty_arguments(p)),
-                             b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA) }
+    | 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block {
+        _Py_FunctionDef(n->v.Name.id,
+                        (params) ? params : CHECK(_PyPegen_empty_arguments(p)),
+                        b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA) }
+    | ASYNC 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block {
+        CHECK_VERSION(
+            5,
+            "Async functions are",
+            _Py_AsyncFunctionDef(n->v.Name.id,
+                            (params) ? params : CHECK(_PyPegen_empty_arguments(p)),
+                            b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA)
+        ) }
 func_type_comment[PyObject*]:
     | NEWLINE t=TYPE_COMMENT &(NEWLINE INDENT) { t }  # Must be followed by indented block
     | invalid_double_type_comments
@@ -399,7 +417,7 @@ term[expr_ty]:
     | a=term '/' b=factor { _Py_BinOp(a, Div, b, EXTRA) }
     | a=term '//' b=factor { _Py_BinOp(a, FloorDiv, b, EXTRA) }
     | a=term '%' b=factor { _Py_BinOp(a, Mod, b, EXTRA) }
-    | a=term '@' b=factor { _Py_BinOp(a, MatMult, b, EXTRA) }
+    | a=term '@' b=factor { CHECK_VERSION(5, "The '@' operator is", _Py_BinOp(a, MatMult, b, EXTRA)) }
     | factor
 factor[expr_ty] (memo):
     | '+' a=factor { _Py_UnaryOp(UAdd, a, EXTRA) }
@@ -410,7 +428,7 @@ power[expr_ty]:
     | a=await_primary '**' b=factor { _Py_BinOp(a, Pow, b, EXTRA) }
     | await_primary
 await_primary[expr_ty] (memo):
-    | AWAIT a=primary { _Py_Await(a, EXTRA) }
+    | AWAIT a=primary { CHECK_VERSION(5, "Await expressions are", _Py_Await(a, EXTRA)) }
     | primary
 primary[expr_ty]:
     | a=primary '.' b=NAME { _Py_Attribute(a, b->v.Name.id, Load, EXTRA) }
@@ -469,8 +487,12 @@ kvpair[KeyValuePair*]:
     | '**' a=bitwise_or { _PyPegen_key_value_pair(p, NULL, a) }
     | a=expression ':' b=expression { _PyPegen_key_value_pair(p, a, b) }
 for_if_clauses[asdl_seq*]:
-    | a=(y=[ASYNC] 'for' a=star_targets 'in' b=disjunction c=('if' z=disjunction { z })*
-        { _Py_comprehension(a, b, c, y != NULL, p->arena) })+ { a }
+    | for_if_clause+
+for_if_clause[comprehension_ty]:
+    | ASYNC 'for' a=star_targets 'in' b=disjunction c=('if' z=disjunction { z })* {
+        CHECK_VERSION(6, "Async comprehensions are", _Py_comprehension(a, b, c, 1, p->arena)) }
+    | 'for' a=star_targets 'in' b=disjunction c=('if' z=disjunction { z })* {
+        _Py_comprehension(a, b, c, 0, p->arena) }
 
 yield_expr[expr_ty]:
     | 'yield' 'from' a=expression { _Py_YieldFrom(a, EXTRA) }

Lib/test/test_type_comments.py

@@ -252,7 +252,6 @@ def test_funcdef(self):
         self.assertEqual(tree.body[0].type_comment, None)
         self.assertEqual(tree.body[1].type_comment, None)
 
-    @support.skip_if_new_parser("Pegen does not support feature_version yet")
     def test_asyncdef(self):
         for tree in self.parse_all(asyncdef, minver=5):
             self.assertEqual(tree.body[0].type_comment, "() -> int")
@@ -261,27 +260,22 @@ def test_asyncdef(self):
         self.assertEqual(tree.body[0].type_comment, None)
         self.assertEqual(tree.body[1].type_comment, None)
 
-    @support.skip_if_new_parser("Pegen does not support feature_version yet")
     def test_asyncvar(self):
         for tree in self.parse_all(asyncvar, maxver=6):
             pass
 
-    @support.skip_if_new_parser("Pegen does not support feature_version yet")
     def test_asynccomp(self):
         for tree in self.parse_all(asynccomp, minver=6):
             pass
 
-    @support.skip_if_new_parser("Pegen does not support feature_version yet")
     def test_matmul(self):
         for tree in self.parse_all(matmul, minver=5):
             pass
 
-    @support.skip_if_new_parser("Pegen does not support feature_version yet")
     def test_fstring(self):
         for tree in self.parse_all(fstring, minver=6):
             pass
 
-    @support.skip_if_new_parser("Pegen does not support feature_version yet")
     def test_underscorednumber(self):
         for tree in self.parse_all(underscorednumber, minver=6):
             pass