bpo-10544: Disallow "yield" in comprehensions and generator expressions. (GH-4564)

Author: serhiy-storchaka

Doc/reference/expressions.rst

@@ -196,8 +196,7 @@ they may depend on the values obtained from the leftmost iterable. For example:
 
 To ensure the comprehension always results in a container of the appropriate
 type, ``yield`` and ``yield from`` expressions are prohibited in the implicitly
-nested scope (in Python 3.7, such expressions emit :exc:`DeprecationWarning`
-when compiled, in Python 3.8+ they will emit :exc:`SyntaxError`).
+nested scope.
 
 Since Python 3.6, in an :keyword:`async def` function, an :keyword:`async for`
 clause may be used to iterate over a :term:`asynchronous iterator`.
@@ -214,8 +213,8 @@ See also :pep:`530`.
 .. versionadded:: 3.6
    Asynchronous comprehensions were introduced.
 
-.. deprecated:: 3.7
-   ``yield`` and ``yield from`` deprecated in the implicitly nested scope.
+.. versionchanged:: 3.8
+   ``yield`` and ``yield from`` prohibited in the implicitly nested scope.
 
 
 .. _lists:
@@ -350,9 +349,7 @@ The parentheses can be omitted on calls with only one argument.  See section
 
 To avoid interfering with the expected operation of the generator expression
 itself, ``yield`` and ``yield from`` expressions are prohibited in the
-implicitly defined generator (in Python 3.7, such expressions emit
-:exc:`DeprecationWarning` when compiled, in Python 3.8+ they will emit
-:exc:`SyntaxError`).
+implicitly defined generator.
 
 If a generator expression contains either :keyword:`async for`
 clauses or :keyword:`await` expressions it is called an
@@ -368,8 +365,8 @@ which is an asynchronous iterator (see :ref:`async-iterators`).
    only appear in :keyword:`async def` coroutines.  Starting
    with 3.7, any function can use asynchronous generator expressions.
 
-.. deprecated:: 3.7
-   ``yield`` and ``yield from`` deprecated in the implicitly nested scope.
+.. versionchanged:: 3.8
+   ``yield`` and ``yield from`` prohibited in the implicitly nested scope.
 
 
 .. _yieldexpr:
@@ -401,12 +398,10 @@ coroutine function to be an asynchronous generator. For example::
 
 Due to their side effects on the containing scope, ``yield`` expressions
 are not permitted as part of the implicitly defined scopes used to
-implement comprehensions and generator expressions (in Python 3.7, such
-expressions emit :exc:`DeprecationWarning` when compiled, in Python 3.8+
-they will emit :exc:`SyntaxError`)..
+implement comprehensions and generator expressions.
 
-.. deprecated:: 3.7
-   Yield expressions deprecated in the implicitly nested scopes used to
+.. versionchanged:: 3.8
+   Yield expressions prohibited in the implicitly nested scopes used to
    implement comprehensions and generator expressions.
 
 Generator functions are described below, while asynchronous generator

Doc/whatsnew/3.8.rst

@@ -113,6 +113,15 @@ This section lists previously described changes and other bugfixes
 that may require changes to your code.
 
 
+Changes in Python behavior
+--------------------------
+
+* Yield expressions (both ``yield`` and ``yield from`` clauses) are now disallowed
+  in comprehensions and generator expressions (aside from the iterable expression
+  in the leftmost :keyword:`for` clause).
+  (Contributed by Serhiy Storchaka in :issue:`10544`.)
+
+
 Changes in the Python API
 -------------------------
 

Lib/test/support/__init__.py

@@ -1061,8 +1061,8 @@ def make_bad_fd():
         file.close()
         unlink(TESTFN)
 
-def check_syntax_error(testcase, statement, *, lineno=None, offset=None):
-    with testcase.assertRaises(SyntaxError) as cm:
+def check_syntax_error(testcase, statement, errtext='', *, lineno=None, offset=None):
+    with testcase.assertRaisesRegex(SyntaxError, errtext) as cm:
         compile(statement, '<test string>', 'exec')
     err = cm.exception
     testcase.assertIsNotNone(err.lineno)

Lib/test/test_grammar.py

@@ -251,6 +251,8 @@ def __getitem__(self, item):
 
 class GrammarTests(unittest.TestCase):
 
+    check_syntax_error = check_syntax_error
+
     # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
     # XXX can't test in a script -- this rule is only used when interactive
 
@@ -920,15 +922,7 @@ def test_yield_in_comprehensions(self):
         def g(): [x for x in [(yield 1)]]
         def g(): [x for x in [(yield from ())]]
 
-        def check(code, warntext):
-            with self.assertWarnsRegex(DeprecationWarning, warntext):
-                compile(code, '<test string>', 'exec')
-            import warnings
-            with warnings.catch_warnings():
-                warnings.filterwarnings('error', category=DeprecationWarning)
-                with self.assertRaisesRegex(SyntaxError, warntext):
-                    compile(code, '<test string>', 'exec')
-
+        check = self.check_syntax_error
         check("def g(): [(yield x) for x in ()]",
               "'yield' inside list comprehension")
         check("def g(): [x for x in () if not (yield x)]",

Misc/NEWS.d/next/Core and Builtins/2017-11-26-00-59-22.bpo-10544.fHOM3V.rst

@@ -0,0 +1,2 @@
+Yield expressions are now disallowed in comprehensions and generator
+expressions except the expression for the outermost iterable.

Python/symtable.c

@@ -1754,35 +1754,18 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e,
         VISIT(st, expr, value);
     VISIT(st, expr, elt);
     if (st->st_cur->ste_generator) {
-        PyObject *msg = PyUnicode_FromString(
+        PyErr_SetString(PyExc_SyntaxError,
             (e->kind == ListComp_kind) ? "'yield' inside list comprehension" :
             (e->kind == SetComp_kind) ? "'yield' inside set comprehension" :
             (e->kind == DictComp_kind) ? "'yield' inside dict comprehension" :
             "'yield' inside generator expression");
-        if (msg == NULL) {
-            symtable_exit_block(st, (void *)e);
-            return 0;
-        }
-        if (PyErr_WarnExplicitObject(PyExc_DeprecationWarning,
-                msg, st->st_filename, st->st_cur->ste_lineno,
-                NULL, NULL) == -1)
-        {
-            if (PyErr_ExceptionMatches(PyExc_DeprecationWarning)) {
-                /* Replace the DeprecationWarning exception with a SyntaxError
-                   to get a more accurate error report */
-                PyErr_Clear();
-                PyErr_SetObject(PyExc_SyntaxError, msg);
-                PyErr_SyntaxLocationObject(st->st_filename,
-                                           st->st_cur->ste_lineno,
-                                           st->st_cur->ste_col_offset);
-            }
-            Py_DECREF(msg);
-            symtable_exit_block(st, (void *)e);
-            return 0;
-        }
-        Py_DECREF(msg);
+        PyErr_SyntaxLocationObject(st->st_filename,
+                                   st->st_cur->ste_lineno,
+                                   st->st_cur->ste_col_offset);
+        symtable_exit_block(st, (void *)e);
+        return 0;
     }
-    st->st_cur->ste_generator |= is_generator;
+    st->st_cur->ste_generator = is_generator;
     return symtable_exit_block(st, (void *)e);
 }