From ca6cd26cdc8299fafdd0c35ca3deadfa62197e7a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 26 Nov 2017 01:00:49 +0200 Subject: [PATCH 1/4] bpo-10544: Disallow "yield" in comprehensions and generator expressions. --- Doc/reference/expressions.rst | 9 +++++++++ Doc/whatsnew/3.7.rst | 6 ++++++ Lib/test/test_generators.py | 8 +++----- Lib/test/test_grammar.py | 15 +++++++++++++++ .../2017-11-26-00-59-22.bpo-10544.fHOM3V.rst | 2 ++ Python/symtable.c | 12 +++++++++++- 6 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-11-26-00-59-22.bpo-10544.fHOM3V.rst diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 1cff8a52df959d5..7cbb81ea61ad48a 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -186,6 +186,9 @@ each time the innermost block is reached. Note that the comprehension is executed in a separate scope, so names assigned to in the target list don't "leak" into the enclosing scope. +Yield expressions are not allowed in comprehensions +except the expression for the outermost iterable. + Since Python 3.6, in an :keyword:`async def` function, an :keyword:`async for` clause may be used to iterate over a :term:`asynchronous iterator`. A comprehension in an :keyword:`async def` function may consist of either a @@ -326,6 +329,9 @@ range(10) for y in bar(x))``. The parentheses can be omitted on calls with only one argument. See section :ref:`calls` for details. +Yield expressions are not allowed in generator expressions +except the expression for the outermost iterable. + If a generator expression contains either :keyword:`async for` clauses or :keyword:`await` expressions it is called an :dfn:`asynchronous generator expression`. An asynchronous generator @@ -364,6 +370,9 @@ coroutine function to be an asynchronous generator. For example:: async def agen(): # defines an asynchronous generator function (PEP 525) yield 123 +Yield expressions are not allowed in comprehensions and generator expressions +except the expression for the outermost iterable. + Generator functions are described below, while asynchronous generator functions are described separately in section :ref:`asynchronous-generator-functions`. diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 514c3c293c080c5..1de9c98d433e33a 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -673,6 +673,12 @@ Changes in Python behavior parentheses can be omitted only on calls. (Contributed by Serhiy Storchaka in :issue:`32012` and :issue:`32023`.) +* Yield expressions now are not allowed in comprehensions and generator + expressions except the expression for the outermost iterable since + comprehensions and generator expressions are implemented using + an internal function. + (Contributed by Serhiy Storchaka in :issue:`10544`.) + Changes in the Python API ------------------------- diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 7eac9d076be4625..92118ad7acf067c 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -1834,7 +1834,9 @@ def printsolution(self, x): An obscene abuse of a yield expression within a generator expression: >>> list((yield 21) for i in range(4)) -[21, None, 21, None, 21, None, 21, None] +Traceback (most recent call last): + ... +SyntaxError: 'yield' inside generator expression And a more sane, but still weird usage: @@ -2106,10 +2108,6 @@ def printsolution(self, x): >>> type(f()) ->>> def f(): x=(i for i in (yield) if (yield)) ->>> type(f()) - - >>> def f(d): d[(yield "a")] = d[(yield "b")] = 27 >>> data = [1,2] >>> g = f(data) diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 65e26bfd383823e..9cb20f2c7e76375 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -841,6 +841,21 @@ def g(): f((yield from ()), 1) # Check annotation refleak on SyntaxError check_syntax_error(self, "def g(a:(yield)): pass") + def test_yield_in_comprehensions(self): + # Check yield in comprehensions + def g(): [x for x in [(yield 1)]] + def g(): [x for x in [(yield from ())]] + check_syntax_error(self, "def g(): [(yield x) for x in ()]") + check_syntax_error(self, "def g(): [x for x in () if not (yield x)]") + check_syntax_error(self, "def g(): [y for x in () for y in [(yield x)]]") + check_syntax_error(self, "def g(): {(yield x) for x in ()}") + check_syntax_error(self, "def g(): {(yield x): x for x in ()}") + check_syntax_error(self, "def g(): {x: (yield x) for x in ()}") + check_syntax_error(self, "def g(): ((yield x) for x in ())") + check_syntax_error(self, "def g(): [(yield from x) for x in ()]") + check_syntax_error(self, "class C: [(yield x) for x in ()]") + check_syntax_error(self, "[(yield x) for x in ()]") + def test_raise(self): # 'raise' test [',' test] try: raise RuntimeError('just testing') diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-26-00-59-22.bpo-10544.fHOM3V.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-26-00-59-22.bpo-10544.fHOM3V.rst new file mode 100644 index 000000000000000..812e7b42f523aae --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-11-26-00-59-22.bpo-10544.fHOM3V.rst @@ -0,0 +1,2 @@ +Yield expressions now are disallowed in comprehensions and generator +expressions except the expression for the outermost iterable. diff --git a/Python/symtable.c b/Python/symtable.c index 55815c91cc9cabd..3ef9ed16815d4b3 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1734,7 +1734,6 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, e->lineno, e->col_offset)) { return 0; } - st->st_cur->ste_generator = is_generator; if (outermost->is_async) { st->st_cur->ste_coroutine = 1; } @@ -1754,6 +1753,17 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, if (value) VISIT(st, expr, value); VISIT(st, expr, elt); + if (st->st_cur->ste_generator) { + PyErr_SetString(PyExc_SyntaxError, + (e->kind == ListComp_kind) ? "'yield' inside list comprehension" : + (e->kind == SetComp_kind) ? "'yield' inside set comprehension" : + (e->kind == DictComp_kind) ? "'yield' inside dict comprehension" : + "'yield' inside generator expression"); + PyErr_SyntaxLocationObject(st->st_filename, + st->st_cur->ste_lineno, + st->st_cur->ste_col_offset); + } + st->st_cur->ste_generator = is_generator; return symtable_exit_block(st, (void *)e); } From 403e84b46903054cdb20480d640f9b28334ef147 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 1 Feb 2018 14:17:39 +0200 Subject: [PATCH 2/4] Add a What's New entry. --- Doc/whatsnew/3.8.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 59f18838d4dbd4a..38f0e9274500b0b 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -113,3 +113,10 @@ This section lists previously described changes and other bugfixes that may require changes to your code. +Changes in Python behavior +-------------------------- + +* Yield expressions (both ``yield`` and ``yield from`` clauses) are now disallowed + in comprehensions and generator expressions (aside from the iterable expression + in the leftmost :keyword:`for` clause). + (Contributed by Serhiy Storchaka in :issue:`10544`.) From 2e14b306ce10871b83bd5dc626905cb0e1f467b0 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 1 Feb 2018 19:18:42 +0200 Subject: [PATCH 3/4] Fix tests. --- Lib/test/test_grammar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 20fd9d0e8a3b84b..d89bfdc063352db 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -922,7 +922,7 @@ def test_yield_in_comprehensions(self): def g(): [x for x in [(yield 1)]] def g(): [x for x in [(yield from ())]] - check = check_syntax_error + check = self.check_syntax_error check("def g(): [(yield x) for x in ()]", "'yield' inside list comprehension") check("def g(): [x for x in () if not (yield x)]", From 9a385c6a4f355271a223d2f1b7cecbb1f4cd677a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 4 Feb 2018 10:16:46 +0200 Subject: [PATCH 4/4] Update 2017-11-26-00-59-22.bpo-10544.fHOM3V.rst --- .../Core and Builtins/2017-11-26-00-59-22.bpo-10544.fHOM3V.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-26-00-59-22.bpo-10544.fHOM3V.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-26-00-59-22.bpo-10544.fHOM3V.rst index 812e7b42f523aae..404f12cbbb31de8 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2017-11-26-00-59-22.bpo-10544.fHOM3V.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2017-11-26-00-59-22.bpo-10544.fHOM3V.rst @@ -1,2 +1,2 @@ -Yield expressions now are disallowed in comprehensions and generator +Yield expressions are now disallowed in comprehensions and generator expressions except the expression for the outermost iterable.