fix an ambiguity in the grammar from the implementation of extended unpacking

(one which was strangely "resolved" by pgen)

This also kills the unused testlist1 rule and fixes parse tree validation of
extended unpacking.

Author: benjaminp

Grammar/Grammar

@@ -37,8 +37,9 @@ stmt: simple_stmt | compound_stmt
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
              import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
-expr_stmt: testlist (augassign (yield_expr|testlist) |
-                     ('=' (yield_expr|testlist))*)
+expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) |
+                     ('=' (yield_expr|testlist_star_expr))*)
+testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
 augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
             '<<=' | '>>=' | '**=' | '//=')
 # For normal assignments, additional restrictions enforced by the interpreter
@@ -86,9 +87,9 @@ lambdef_nocond: 'lambda' [varargslist] ':' test_nocond
 or_test: and_test ('or' and_test)*
 and_test: not_test ('and' not_test)*
 not_test: 'not' not_test | comparison
-comparison: star_expr (comp_op star_expr)*
+comparison: expr (comp_op expr)*
 comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
-star_expr: ['*'] expr
+star_expr: '*' expr
 expr: xor_expr ('|' xor_expr)*
 xor_expr: and_expr ('^' and_expr)*
 and_expr: shift_expr ('&' shift_expr)*
@@ -101,12 +102,12 @@ atom: ('(' [yield_expr|testlist_comp] ')' |
        '[' [testlist_comp] ']' |
        '{' [dictorsetmaker] '}' |
        NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False')
-testlist_comp: test ( comp_for | (',' test)* [','] )
+testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 subscriptlist: subscript (',' subscript)* [',']
 subscript: test | [test] ':' [test] [sliceop]
 sliceop: ':' [test]
-exprlist: star_expr (',' star_expr)* [',']
+exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
 testlist: test (',' test)* [',']
 dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
                   (test (comp_for | (',' test)* [','])) )
@@ -123,8 +124,6 @@ comp_iter: comp_for | comp_if
 comp_for: 'for' exprlist 'in' or_test [comp_iter]
 comp_if: 'if' test_nocond [comp_iter]
 
-testlist1: test (',' test)*
-
 # not used in grammar, but may appear in "node" passed from Parser to Compiler
 encoding_decl: NAME
 

Include/graminit.h

@@ -16,68 +16,68 @@
 #define simple_stmt 269
 #define small_stmt 270
 #define expr_stmt 271
-#define augassign 272
-#define del_stmt 273
-#define pass_stmt 274
-#define flow_stmt 275
-#define break_stmt 276
-#define continue_stmt 277
-#define return_stmt 278
-#define yield_stmt 279
-#define raise_stmt 280
-#define import_stmt 281
-#define import_name 282
-#define import_from 283
-#define import_as_name 284
-#define dotted_as_name 285
-#define import_as_names 286
-#define dotted_as_names 287
-#define dotted_name 288
-#define global_stmt 289
-#define nonlocal_stmt 290
-#define assert_stmt 291
-#define compound_stmt 292
-#define if_stmt 293
-#define while_stmt 294
-#define for_stmt 295
-#define try_stmt 296
-#define with_stmt 297
-#define with_item 298
-#define except_clause 299
-#define suite 300
-#define test 301
-#define test_nocond 302
-#define lambdef 303
-#define lambdef_nocond 304
-#define or_test 305
-#define and_test 306
-#define not_test 307
-#define comparison 308
-#define comp_op 309
-#define star_expr 310
-#define expr 311
-#define xor_expr 312
-#define and_expr 313
-#define shift_expr 314
-#define arith_expr 315
-#define term 316
-#define factor 317
-#define power 318
-#define atom 319
-#define testlist_comp 320
-#define trailer 321
-#define subscriptlist 322
-#define subscript 323
-#define sliceop 324
-#define exprlist 325
-#define testlist 326
-#define dictorsetmaker 327
-#define classdef 328
-#define arglist 329
-#define argument 330
-#define comp_iter 331
-#define comp_for 332
-#define comp_if 333
-#define testlist1 334
+#define testlist_star_expr 272
+#define augassign 273
+#define del_stmt 274
+#define pass_stmt 275
+#define flow_stmt 276
+#define break_stmt 277
+#define continue_stmt 278
+#define return_stmt 279
+#define yield_stmt 280
+#define raise_stmt 281
+#define import_stmt 282
+#define import_name 283
+#define import_from 284
+#define import_as_name 285
+#define dotted_as_name 286
+#define import_as_names 287
+#define dotted_as_names 288
+#define dotted_name 289
+#define global_stmt 290
+#define nonlocal_stmt 291
+#define assert_stmt 292
+#define compound_stmt 293
+#define if_stmt 294
+#define while_stmt 295
+#define for_stmt 296
+#define try_stmt 297
+#define with_stmt 298
+#define with_item 299
+#define except_clause 300
+#define suite 301
+#define test 302
+#define test_nocond 303
+#define lambdef 304
+#define lambdef_nocond 305
+#define or_test 306
+#define and_test 307
+#define not_test 308
+#define comparison 309
+#define comp_op 310
+#define star_expr 311
+#define expr 312
+#define xor_expr 313
+#define and_expr 314
+#define shift_expr 315
+#define arith_expr 316
+#define term 317
+#define factor 318
+#define power 319
+#define atom 320
+#define testlist_comp 321
+#define trailer 322
+#define subscriptlist 323
+#define subscript 324
+#define sliceop 325
+#define exprlist 326
+#define testlist 327
+#define dictorsetmaker 328
+#define classdef 329
+#define arglist 330
+#define argument 331
+#define comp_iter 332
+#define comp_for 333
+#define comp_if 334
 #define encoding_decl 335
 #define yield_expr 336

Lib/symbol.py

@@ -26,69 +26,69 @@
 simple_stmt = 269
 small_stmt = 270
 expr_stmt = 271
-augassign = 272
-del_stmt = 273
-pass_stmt = 274
-flow_stmt = 275
-break_stmt = 276
-continue_stmt = 277
-return_stmt = 278
-yield_stmt = 279
-raise_stmt = 280
-import_stmt = 281
-import_name = 282
-import_from = 283
-import_as_name = 284
-dotted_as_name = 285
-import_as_names = 286
-dotted_as_names = 287
-dotted_name = 288
-global_stmt = 289
-nonlocal_stmt = 290
-assert_stmt = 291
-compound_stmt = 292
-if_stmt = 293
-while_stmt = 294
-for_stmt = 295
-try_stmt = 296
-with_stmt = 297
-with_item = 298
-except_clause = 299
-suite = 300
-test = 301
-test_nocond = 302
-lambdef = 303
-lambdef_nocond = 304
-or_test = 305
-and_test = 306
-not_test = 307
-comparison = 308
-comp_op = 309
-star_expr = 310
-expr = 311
-xor_expr = 312
-and_expr = 313
-shift_expr = 314
-arith_expr = 315
-term = 316
-factor = 317
-power = 318
-atom = 319
-testlist_comp = 320
-trailer = 321
-subscriptlist = 322
-subscript = 323
-sliceop = 324
-exprlist = 325
-testlist = 326
-dictorsetmaker = 327
-classdef = 328
-arglist = 329
-argument = 330
-comp_iter = 331
-comp_for = 332
-comp_if = 333
-testlist1 = 334
+testlist_star_expr = 272
+augassign = 273
+del_stmt = 274
+pass_stmt = 275
+flow_stmt = 276
+break_stmt = 277
+continue_stmt = 278
+return_stmt = 279
+yield_stmt = 280
+raise_stmt = 281
+import_stmt = 282
+import_name = 283
+import_from = 284
+import_as_name = 285
+dotted_as_name = 286
+import_as_names = 287
+dotted_as_names = 288
+dotted_name = 289
+global_stmt = 290
+nonlocal_stmt = 291
+assert_stmt = 292
+compound_stmt = 293
+if_stmt = 294
+while_stmt = 295
+for_stmt = 296
+try_stmt = 297
+with_stmt = 298
+with_item = 299
+except_clause = 300
+suite = 301
+test = 302
+test_nocond = 303
+lambdef = 304
+lambdef_nocond = 305
+or_test = 306
+and_test = 307
+not_test = 308
+comparison = 309
+comp_op = 310
+star_expr = 311
+expr = 312
+xor_expr = 313
+and_expr = 314
+shift_expr = 315
+arith_expr = 316
+term = 317
+factor = 318
+power = 319
+atom = 320
+testlist_comp = 321
+trailer = 322
+subscriptlist = 323
+subscript = 324
+sliceop = 325
+exprlist = 326
+testlist = 327
+dictorsetmaker = 328
+classdef = 329
+arglist = 330
+argument = 331
+comp_iter = 332
+comp_for = 333
+comp_if = 334
 encoding_decl = 335
 yield_expr = 336
 #--end constants--

Lib/test/test_parser.py

@@ -242,6 +242,12 @@ def walk(tree):
             (0, '', 2, -1)],
                          terminals)
 
+    def test_extended_unpacking(self):
+        self.check_suite("*a = y")
+        self.check_suite("x, *b, = m")
+        self.check_suite("[*a, *b] = y")
+        self.check_suite("for [*x, b] in x: pass")
+
 
 #
 #  Second, we take *invalid* trees and make sure we get ParserError

Misc/NEWS

@@ -12,6 +12,8 @@ What's New in Python 3.2 Alpha 1?
 Core and Builtins
 -----------------
 
+- Issue #5460: Fix an ambiguity in the grammar.
+
 - Issue #1766304: Improve performance of membership tests on range objects.
 
 - Issue #6713: Improve performance of integer -> string conversions.