bpo-30406: Make async and await proper keywords (#1669)

Per PEP 492, 'async' and 'await' should become proper keywords in 3.7.

Author: JelleZijlstra

Doc/library/token.rst

@@ -98,8 +98,6 @@ The token constants are:
           RARROW
           ELLIPSIS
           OP
-          AWAIT
-          ASYNC
           ERRORTOKEN
           N_TOKENS
           NT_OFFSET
@@ -129,9 +127,11 @@ the :mod:`tokenize` module.
 
 
 .. versionchanged:: 3.5
-   Added :data:`AWAIT` and :data:`ASYNC` tokens. Starting with
-   Python 3.7, "async" and "await" will be tokenized as :data:`NAME`
-   tokens, and :data:`AWAIT` and :data:`ASYNC` will be removed.
+   Added :data:`AWAIT` and :data:`ASYNC` tokens.
 
 .. versionchanged:: 3.7
    Added :data:`COMMENT`, :data:`NL` and :data:`ENCODING` tokens.
+
+.. versionchanged:: 3.7
+   Removed :data:`AWAIT` and :data:`ASYNC` tokens. "async" and "await" are
+   now tokenized as :data:`NAME` tokens.

Doc/tools/extensions/pyspecific.py

@@ -272,9 +272,9 @@ def run(self):
 # Support for building "topic help" for pydoc
 
 pydoc_topic_labels = [
-    'assert', 'assignment', 'atom-identifiers', 'atom-literals',
-    'attribute-access', 'attribute-references', 'augassign', 'binary',
-    'bitwise', 'bltin-code-objects', 'bltin-ellipsis-object',
+    'assert', 'assignment', 'async', 'atom-identifiers', 'atom-literals',
+    'attribute-access', 'attribute-references', 'augassign', 'await',
+    'binary', 'bitwise', 'bltin-code-objects', 'bltin-ellipsis-object',
     'bltin-null-object', 'bltin-type-objects', 'booleans',
     'break', 'callable-types', 'calls', 'class', 'comparisons', 'compound',
     'context-managers', 'continue', 'conversions', 'customization', 'debugger',

Grammar/Grammar

@@ -16,7 +16,7 @@ decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
 decorators: decorator+
 decorated: decorators (classdef | funcdef | async_funcdef)
 
-async_funcdef: ASYNC funcdef
+async_funcdef: 'async' funcdef
 funcdef: 'def' NAME parameters ['->' test] ':' suite
 
 parameters: '(' [typedargslist] ')'
@@ -68,7 +68,7 @@ nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
 assert_stmt: 'assert' test [',' test]
 
 compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
-async_stmt: ASYNC (funcdef | with_stmt | for_stmt)
+async_stmt: 'async' (funcdef | with_stmt | for_stmt)
 if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
 while_stmt: 'while' test ':' suite ['else' ':' suite]
 for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
@@ -103,7 +103,7 @@ arith_expr: term (('+'|'-') term)*
 term: factor (('*'|'@'|'/'|'%'|'//') factor)*
 factor: ('+'|'-'|'~') factor | power
 power: atom_expr ['**' factor]
-atom_expr: [AWAIT] atom trailer*
+atom_expr: ['await'] atom trailer*
 atom: ('(' [yield_expr|testlist_comp] ')' |
        '[' [testlist_comp] ']' |
        '{' [dictorsetmaker] '}' |
@@ -139,7 +139,8 @@ argument: ( test [comp_for] |
             '*' test )
 
 comp_iter: comp_for | comp_if
-comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter]
+sync_comp_for: 'for' exprlist 'in' or_test [comp_iter]
+comp_for: ['async'] sync_comp_for
 comp_if: 'if' test_nocond [comp_iter]
 
 # not used in grammar, but may appear in "node" passed from Parser to Compiler

Include/graminit.h

@@ -81,8 +81,9 @@
 #define arglist 334
 #define argument 335
 #define comp_iter 336
-#define comp_for 337
-#define comp_if 338
-#define encoding_decl 339
-#define yield_expr 340
-#define yield_arg 341
+#define sync_comp_for 337
+#define comp_for 338
+#define comp_if 339
+#define encoding_decl 340
+#define yield_expr 341
+#define yield_arg 342

Include/token.h

@@ -9,77 +9,75 @@ extern "C" {
 
 #undef TILDE   /* Prevent clash of our definition with system macro. Ex AIX, ioctl.h */
 
-#define ENDMARKER	0
-#define NAME		1
-#define NUMBER		2
-#define STRING		3
-#define NEWLINE		4
-#define INDENT		5
-#define DEDENT		6
-#define LPAR		7
-#define RPAR		8
-#define LSQB		9
-#define RSQB		10
-#define COLON		11
-#define COMMA		12
-#define SEMI		13
-#define PLUS		14
-#define MINUS		15
-#define STAR		16
-#define SLASH		17
-#define VBAR		18
-#define AMPER		19
-#define LESS		20
-#define GREATER		21
-#define EQUAL		22
-#define DOT		23
-#define PERCENT		24
-#define LBRACE		25
-#define RBRACE		26
-#define EQEQUAL		27
-#define NOTEQUAL	28
-#define LESSEQUAL	29
-#define GREATEREQUAL	30
-#define TILDE		31
-#define CIRCUMFLEX	32
-#define LEFTSHIFT	33
-#define RIGHTSHIFT	34
-#define DOUBLESTAR	35
-#define PLUSEQUAL	36
-#define MINEQUAL	37
-#define STAREQUAL	38
-#define SLASHEQUAL	39
-#define PERCENTEQUAL	40
-#define AMPEREQUAL	41
-#define VBAREQUAL	42
-#define CIRCUMFLEXEQUAL	43
-#define LEFTSHIFTEQUAL	44
-#define RIGHTSHIFTEQUAL	45
-#define DOUBLESTAREQUAL	46
-#define DOUBLESLASH	47
+#define ENDMARKER       0
+#define NAME            1
+#define NUMBER          2
+#define STRING          3
+#define NEWLINE         4
+#define INDENT          5
+#define DEDENT          6
+#define LPAR            7
+#define RPAR            8
+#define LSQB            9
+#define RSQB            10
+#define COLON           11
+#define COMMA           12
+#define SEMI            13
+#define PLUS            14
+#define MINUS           15
+#define STAR            16
+#define SLASH           17
+#define VBAR            18
+#define AMPER           19
+#define LESS            20
+#define GREATER         21
+#define EQUAL           22
+#define DOT             23
+#define PERCENT         24
+#define LBRACE          25
+#define RBRACE          26
+#define EQEQUAL         27
+#define NOTEQUAL        28
+#define LESSEQUAL       29
+#define GREATEREQUAL    30
+#define TILDE           31
+#define CIRCUMFLEX      32
+#define LEFTSHIFT       33
+#define RIGHTSHIFT      34
+#define DOUBLESTAR      35
+#define PLUSEQUAL       36
+#define MINEQUAL        37
+#define STAREQUAL       38
+#define SLASHEQUAL      39
+#define PERCENTEQUAL    40
+#define AMPEREQUAL      41
+#define VBAREQUAL       42
+#define CIRCUMFLEXEQUAL 43
+#define LEFTSHIFTEQUAL  44
+#define RIGHTSHIFTEQUAL 45
+#define DOUBLESTAREQUAL 46
+#define DOUBLESLASH     47
 #define DOUBLESLASHEQUAL 48
 #define AT              49
-#define ATEQUAL		50
+#define ATEQUAL         50
 #define RARROW          51
 #define ELLIPSIS        52
 /* Don't forget to update the table _PyParser_TokenNames in tokenizer.c! */
-#define OP		53
-#define AWAIT		54
-#define ASYNC		55
-#define ERRORTOKEN	56
+#define OP              53
+#define ERRORTOKEN      54
 /* These aren't used by the C tokenizer but are needed for tokenize.py */
-#define COMMENT		57
-#define NL		58
-#define ENCODING		59
-#define N_TOKENS	60
+#define COMMENT         55
+#define NL              56
+#define ENCODING                57
+#define N_TOKENS        58
 
 /* Special definitions for cooperation with parser */
 
-#define NT_OFFSET		256
+#define NT_OFFSET               256
 
-#define ISTERMINAL(x)		((x) < NT_OFFSET)
-#define ISNONTERMINAL(x)	((x) >= NT_OFFSET)
-#define ISEOF(x)		((x) == ENDMARKER)
+#define ISTERMINAL(x)           ((x) < NT_OFFSET)
+#define ISNONTERMINAL(x)        ((x) >= NT_OFFSET)
+#define ISEOF(x)                ((x) == ENDMARKER)
 
 
 PyAPI_DATA(const char *) _PyParser_TokenNames[]; /* Token names */

Lib/keyword.py

@@ -20,6 +20,8 @@
         'and',
         'as',
         'assert',
+        'async',
+        'await',
         'break',
         'class',
         'continue',

Lib/lib2to3/Grammar.txt

@@ -34,7 +34,7 @@ eval_input: testlist NEWLINE* ENDMARKER
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
 decorators: decorator+
 decorated: decorators (classdef | funcdef | async_funcdef)
-async_funcdef: ASYNC funcdef
+async_funcdef: 'async' funcdef
 funcdef: 'def' NAME parameters ['->' test] ':' suite
 parameters: '(' [typedargslist] ')'
 typedargslist: ((tfpdef ['=' test] ',')*
@@ -85,7 +85,7 @@ exec_stmt: 'exec' expr ['in' test [',' test]]
 assert_stmt: 'assert' test [',' test]
 
 compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
-async_stmt: ASYNC (funcdef | with_stmt | for_stmt)
+async_stmt: 'async' (funcdef | with_stmt | for_stmt)
 if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
 while_stmt: 'while' test ':' suite ['else' ':' suite]
 for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
@@ -124,7 +124,7 @@ shift_expr: arith_expr (('<<'|'>>') arith_expr)*
 arith_expr: term (('+'|'-') term)*
 term: factor (('*'|'@'|'/'|'%'|'//') factor)*
 factor: ('+'|'-'|'~') factor | power
-power: [AWAIT] atom trailer* ['**' factor]
+power: ['await'] atom trailer* ['**' factor]
 atom: ('(' [yield_expr|testlist_gexp] ')' |
        '[' [listmaker] ']' |
        '{' [dictsetmaker] '}' |
@@ -161,7 +161,7 @@ argument: ( test [comp_for] |
 	    star_expr )
 
 comp_iter: comp_for | comp_if
-comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter]
+comp_for: ['async'] 'for' exprlist 'in' or_test [comp_iter]
 comp_if: 'if' old_test [comp_iter]
 
 # As noted above, testlist_safe extends the syntax allowed in list
@@ -180,7 +180,7 @@ comp_if: 'if' old_test [comp_iter]
 #
 # See https://bugs.python.org/issue27494
 old_comp_iter: old_comp_for | old_comp_if
-old_comp_for: [ASYNC] 'for' exprlist 'in' testlist_safe [old_comp_iter]
+old_comp_for: ['async'] 'for' exprlist 'in' testlist_safe [old_comp_iter]
 old_comp_if: 'if' old_test [old_comp_iter]
 
 testlist1: test (',' test)*

Lib/lib2to3/pgen2/token.py

@@ -62,10 +62,8 @@
 COMMENT = 53
 NL = 54
 RARROW = 55
-AWAIT = 56
-ASYNC = 57
-ERRORTOKEN = 58
-N_TOKENS = 59
+ERRORTOKEN = 56
+N_TOKENS = 57
 NT_OFFSET = 256
 #--end constants--