From e3e43cbe00ff0b17038a68ca75825f4bfb061366 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 2 Nov 2026 11:49:01 +0100 Subject: [PATCH] gh-161170: Lazily materialize coroutine objects for immediate awaits `await f()` on an `async def` no longer allocates a coroutine object unless the body actually suspends. GET_AWAITABLE now also pushes the NULL and None that start the SEND loop, so the compiler emits a single instruction between an awaited expression and SEND. RETURN_GENERATOR uses that opcode in the caller to recognize an immediately awaited call: the body then keeps running in the already-pushed thread frame (marked lazy_coro, with its own exception-state item on the data stack) and the caller's frame is set up as SEND_GEN would leave it. A body that returns or raises without suspending never creates a coroutine object; YIELD_VALUE materializes one when the body first suspends, sized from the frame's code object. The eager path is kept for callers from C, instrumented SEND/END_SEND, coroutine origin tracking, JIT trace recording and tier 2. frame.f_generator and PyFrame_GetGenerator() return None/NULL for such a coroutine until it first suspends; asyncio.graph classifies frames by co_flags instead. Co-Authored-By: Claude Fable 5.1 --- Doc/c-api/frame.rst | 4 + Doc/library/dis.rst | 7 +- Doc/library/inspect.rst | 4 +- Doc/reference/datamodel.rst | 14 +- Doc/whatsnew/3.16.rst | 33 ++ Include/internal/pycore_ceval.h | 8 + Include/internal/pycore_interpframe.h | 4 + Include/internal/pycore_interpframe_structs.h | 6 +- Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 4 +- Include/internal/pycore_uop_ids.h | 2 +- Include/internal/pycore_uop_metadata.h | 8 +- InternalDocs/generators.md | 16 +- Lib/asyncio/graph.py | 63 +-- Lib/test/test_asyncio/test_graph.py | 47 +- Lib/test/test_coroutines.py | 504 ++++++++++++++++++ Lib/test/test_dis.py | 8 +- Lib/test/test_opcache.py | 7 + ...-10-27-12-00-00.gh-issue-161170.lazyCo.rst | 3 + Modules/_testinternalcapi/test_cases.c.h | 165 ++++-- Objects/genobject.c | 81 ++- Python/bytecodes.c | 132 ++++- Python/ceval.c | 98 ++++ Python/codegen.c | 15 +- Python/executor_cases.c.h | 138 +++-- Python/generated_cases.c.h | 165 ++++-- Python/intrinsics.c | 6 +- Python/optimizer_cases.c.h | 9 + Tools/cases_generator/analyzer.py | 1 + 29 files changed, 1330 insertions(+), 225 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-27-12-00-00.gh-issue-161170.lazyCo.rst diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 4159ff6e5965fbd..19f58bc80b7d407 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -85,6 +85,10 @@ See also :ref:`Reflection `. .. versionadded:: 3.11 + .. versionchanged:: 3.16 + May return ``NULL`` for a coroutine frame before its first suspension; + see :attr:`frame.f_generator`. + .. c:function:: PyObject* PyFrame_GetGlobals(PyFrameObject *frame) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 15c138ee938bf66..1985cbb92190be2 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -806,7 +806,8 @@ not have to be) the original ``STACK[-2]``. Implements ``STACK[-1] = get_awaitable(STACK[-1])``, where ``get_awaitable(o)`` returns ``o`` if ``o`` is a coroutine object or a generator object with the :data:`~inspect.CO_ITERABLE_COROUTINE` flag, or resolves - ``o.__await__``. + ``o.__await__``. Then pushes ``NULL`` and ``None``, the initial stack + layout for the :opcode:`SEND` loop that follows. If the ``where`` operand is nonzero, it indicates where the instruction occurs: @@ -819,6 +820,10 @@ not have to be) the original ``STACK[-2]``. .. versionchanged:: 3.11 Previously, this instruction did not have an oparg. + .. versionchanged:: 3.16 + Also pushes ``NULL`` and ``None``; previously the compiler emitted + :opcode:`PUSH_NULL` and :opcode:`LOAD_COMMON_CONSTANT` after it. + .. opcode:: GET_AITER diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 29bdae28b1fe268..a33c12ec515c089 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -151,7 +151,9 @@ attributes (see :ref:`import-mod-attrs` for module attributes): | | | coroutine object that | | | | owns this frame, or | | | | ``None`` if the frame is | -| | | of a regular function | +| | | of a regular function or | +| | | the generator is not | +| | | materialized yet | +-----------------+-------------------+---------------------------+ | | f_trace | tracing function for this | | | | frame, or ``None`` | diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index d2089724fa29572..2079d009dce6191 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1736,10 +1736,22 @@ Special read-only attributes * - .. attribute:: frame.f_generator - The :term:`generator` or :term:`coroutine` object that owns this frame, - or ``None`` if the frame is a normal function. + or ``None`` if the frame is a normal function or if the generator + is not materialized yet. + + .. impl-detail:: + + A coroutine that is awaited as soon as it is called + (``await coro_func()``) may run without a coroutine object until + it first suspends, so this attribute is ``None`` before that. + Use :attr:`~codeobject.co_flags` of :attr:`frame.f_code` to tell + coroutine frames apart from normal function frames. .. versionadded:: 3.14 + .. versionchanged:: 3.16 + May be ``None`` for a coroutine frame before its first suspension. + .. index:: single: f_trace (frame attribute) single: f_trace_lines (frame attribute) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index e342aa71c7924ea..d64bfd58fc1c336 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -1015,6 +1015,19 @@ datetime and time (Contributed by Paul Ganssle in :gh:`158252`, based on work by Stan Ulbrych.) +Interpreter +----------- + +* A coroutine that is awaited as soon as it is called + (``await coro_func()``) now runs without creating a coroutine object + unless it suspends, making awaits that complete immediately about 40% + faster in a micro-benchmark (``await noop()``: 69 ns to 41 ns on x86-64). + This does not apply while coroutine origin tracking is enabled or inside + code compiled by the experimental JIT. + Until it first suspends, :attr:`frame.f_generator` is ``None`` for such a + coroutine; see :ref:`whatsnew316-porting`. + (Contributed by Andrew Svetlov in :gh:`161170`.) + re -- @@ -1233,12 +1246,23 @@ New deprecations .. include:: ../deprecations/pending-removal-in-future.rst +.. _whatsnew316-porting: + Porting to Python 3.16 ====================== This section lists previously described changes and other bugfixes that may require changes to your code. +* :attr:`frame.f_generator` and :c:func:`PyFrame_GetGenerator` may now + return ``None``/``NULL`` for a coroutine frame before its first + suspension, because a coroutine that is awaited as soon as it is called + (``await coro_func()``) runs without a coroutine object until then. + Code that used ``frame.f_generator is None`` to detect frames of normal + functions should check ``frame.f_code.co_flags`` for + :data:`~inspect.CO_COROUTINE` instead. + (Contributed by Andrew Svetlov in :gh:`161170`.) + * In :mod:`tkinter`, the *name* parameter of the :meth:`~tkinter.Misc.wait_variable`, :meth:`~tkinter.Misc.setvar` and :meth:`~tkinter.Misc.getvar` methods and the *value* parameter of @@ -1294,6 +1318,15 @@ that may require changes to your code. (Contributed by An Long in :gh:`86768`.) +CPython bytecode changes +======================== + +* :opcode:`GET_AWAITABLE` now also pushes the ``NULL`` and ``None`` that + start the :opcode:`SEND` loop following it, so the compiler no longer + emits :opcode:`PUSH_NULL` and :opcode:`LOAD_COMMON_CONSTANT` after it. + (Contributed by Andrew Svetlov in :gh:`161170`.) + + Build changes ============= diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index a0fea389b6502c0..e7528d13e67c1c9 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -267,6 +267,14 @@ extern int _PyEval_EnsureBuiltinsWithModule( PyObject **p_builtins); PyAPI_FUNC(PyObject *)_Py_MakeCoro(PyFunctionObject *func); +// Lazily created coroutines (see RETURN_GENERATOR in bytecodes.c). +// Exported for _testinternalcapi and external JIT support. +PyAPI_FUNC(PyObject *) _PyCoro_MaterializeLazy(_PyInterpreterFrame *frame); +PyAPI_FUNC(_Py_CODEUNIT *) _PyEval_LazyCoroSendInstr(PyThreadState *tstate, + _PyInterpreterFrame *frame); +PyAPI_FUNC(_PyInterpreterFrame *) _PyEval_MoveLazyCoroFrame(PyThreadState *tstate, + PyGenObject *gen, + _PyInterpreterFrame *frame); /* Handle signals, pending calls, GIL drop request and asynchronous exception */ diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index a85fa9bd32853c8..ea280a324e758a8 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -152,6 +152,8 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * dest->stackpointer = dest->localsplus + stacktop; // visited is GC bookkeeping for the current stack walk, not frame state. dest->visited = 0; + // A lazily started coroutine stops being lazy once its frame is copied. + dest->lazy_coro = 0; #ifdef Py_DEBUG dest->stackpointer_valid = src->stackpointer_valid; dest->lltrace = src->lltrace; @@ -208,6 +210,7 @@ _PyFrame_Initialize( frame->return_offset = 0; frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; + frame->lazy_coro = 0; #ifdef Py_DEBUG frame->stackpointer_valid = 1; frame->lltrace = 0; @@ -441,6 +444,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int #endif frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; + frame->lazy_coro = 0; #ifdef Py_DEBUG frame->stackpointer_valid = 1; frame->lltrace = 0; diff --git a/Include/internal/pycore_interpframe_structs.h b/Include/internal/pycore_interpframe_structs.h index 0d6a37cc2c1a17a..0b593623dc09a85 100644 --- a/Include/internal/pycore_interpframe_structs.h +++ b/Include/internal/pycore_interpframe_structs.h @@ -44,9 +44,13 @@ struct _PyInterpreterFrame { char owner; uint8_t visited:1; uint8_t external:1; /* Set when the frame was pushed by something other than the VM */ + /* Set for a thread-owned frame running a coroutine body whose coroutine + * object has not been created yet (see RETURN_GENERATOR in bytecodes.c). + * Its own _PyErr_StackItem lives on the data stack right after it. */ + uint8_t lazy_coro:1; #ifdef Py_DEBUG uint8_t stackpointer_valid:1; - uint8_t lltrace:5; + uint8_t lltrace:4; #endif /* Locals and stack */ _PyStackRef localsplus[1]; diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 15d3ad6f3a336b7..f199d408b9f287f 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -304,6 +304,7 @@ Known values: Python 3.16a1 3704 (Replace DELETE_ATTR with PUSH_NULL; STORE_ATTR) Python 3.16a1 3705 (Add INTRINSIC_ADD_CONDITIONAL_ANNOTATION) Python 3.16a1 3706 (Add INTRINSIC_MATCH_CLASS_ISINSTANCE) + Python 3.16a1 3707 (GET_AWAITABLE pushes NULL and None for SEND) Python 3.17 will start with 3750 @@ -313,7 +314,7 @@ Known values: */ -#define PYC_MAGIC_NUMBER 3706 +#define PYC_MAGIC_NUMBER 3707 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 04b826ed09144e7..9b34861ec41cd5e 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -716,7 +716,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case GET_ANEXT: return 2; case GET_AWAITABLE: - return 1; + return 3; case GET_ITER: return 2; case GET_ITER_SELF: @@ -1330,7 +1330,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [ANNOTATIONS_PLACEHOLDER] = { true, -1, HAS_PURE_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index ba6e0252024a6df..8006830801916a9 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -690,7 +690,7 @@ enum { _FOR_ITER_VIRTUAL_TIER_TWO_r23, _GET_AITER_r11, _GET_ANEXT_r12, - _GET_AWAITABLE_r11, + _GET_AWAITABLE_r13, _GET_ITER_r12, _GET_ITER_TRAD_r12, _GET_LEN_r12, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 0cd82c53aa932ae..6c1560eb9cb48da 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -174,7 +174,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SEND_VIRTUAL_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, [_GUARD_3OS_ASYNC_GEN_ASEND] = HAS_EXIT_FLAG, [_SEND_ASYNC_GEN_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -1639,7 +1639,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 1, 1, _GET_AWAITABLE_r11 }, + { 3, 1, _GET_AWAITABLE_r13 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -4284,7 +4284,7 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_RETURN_VALUE_r11] = _RETURN_VALUE, [_GET_AITER_r11] = _GET_AITER, [_GET_ANEXT_r12] = _GET_ANEXT, - [_GET_AWAITABLE_r11] = _GET_AWAITABLE, + [_GET_AWAITABLE_r13] = _GET_AWAITABLE, [_SEND_GEN_FRAME_r33] = _SEND_GEN_FRAME, [_GUARD_TOS_IS_NONE_r01] = _GUARD_TOS_IS_NONE, [_GUARD_TOS_IS_NONE_r11] = _GUARD_TOS_IS_NONE, @@ -5238,7 +5238,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GET_ANEXT] = "_GET_ANEXT", [_GET_ANEXT_r12] = "_GET_ANEXT_r12", [_GET_AWAITABLE] = "_GET_AWAITABLE", - [_GET_AWAITABLE_r11] = "_GET_AWAITABLE_r11", + [_GET_AWAITABLE_r13] = "_GET_AWAITABLE_r13", [_GET_ITER] = "_GET_ITER", [_GET_ITER_r12] = "_GET_ITER_r12", [_GET_ITER_TRAD] = "_GET_ITER_TRAD", diff --git a/InternalDocs/generators.md b/InternalDocs/generators.md index fa652cd231cfa95..07a713e09846ea5 100644 --- a/InternalDocs/generators.md +++ b/InternalDocs/generators.md @@ -45,7 +45,21 @@ The generator's frame is initialized as a copy of the frame in which `RETURN_GENERATOR` is executing, but its `owner` field is overwritten to indicate that it is owned by a generator. Finally, `RETURN_GENERATOR` pushes the new generator object to the stack and returns to the caller of the generator function (at -which time its frame is destroyed). When the generator is next resumed by +which time its frame is destroyed). + +There is one exception: a coroutine whose caller is about to await it right +away (`await f()`, i.e. the caller's next instruction is `GET_AWAITABLE`, +which the compiler only emits directly before a `SEND` loop) keeps running in the frame +in which `RETURN_GENERATOR` is executing, with the `lazy_coro` flag set and its +own `_PyErr_StackItem` pushed on the data stack right after the frame. +The caller's frame is set up as if `SEND_GEN` had pushed the coroutine frame +(`[None, NULL]` on its stack, instruction pointer at the `SEND`), so that a body +that returns or raises without suspending never creates a coroutine object. +If the body reaches `YIELD_VALUE`, the coroutine object is created then and +the frame is moved into it (see `_PyCoro_MaterializeLazy()` and +`_PyEval_MoveLazyCoroFrame()`). + +When the generator is next resumed by [`gen_send_ex2()`](../Objects/genobject.c), `_PyEval_EvalFrame()` is called to continue executing the generator function, in the frame that is embedded in the generator object. diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py index b5c9dafe8ed50a1..37bcf7d82ae8e39 100644 --- a/Lib/asyncio/graph.py +++ b/Lib/asyncio/graph.py @@ -1,6 +1,7 @@ """Introspection utils for tasks call graphs.""" import dataclasses +import inspect import io import sys import types @@ -37,6 +38,26 @@ class FutureCallGraph: awaited_by: tuple["FutureCallGraph", ...] +# A coroutine that is awaited right away runs without a coroutine +# object until it first suspends, so ``f.f_generator`` may be None +# for a coroutine frame; classify frames by their code instead. +_CO_ASYNC_FLAGS = inspect.CO_COROUTINE | inspect.CO_ASYNC_GENERATOR +_CO_GEN_FLAGS = _CO_ASYNC_FLAGS | inspect.CO_GENERATOR +_FRAME_TAGS = { + inspect.CO_COROUTINE: 'async ', + inspect.CO_ASYNC_GENERATOR: 'async generator ', + inspect.CO_GENERATOR: 'generator ', +} + + +def _is_async_frame(f): + return bool(f.f_code.co_flags & _CO_ASYNC_FLAGS) + + +def _is_sync_frame(f): + return not (f.f_code.co_flags & _CO_GEN_FLAGS) + + def _build_graph_for_future( future: futures.Future, *, @@ -161,12 +182,10 @@ def capture_call_graph( try: while f is not None: # gh-156988: sync gen should not clear the call chain - is_async = isinstance( - f.f_generator, (types.CoroutineType, types.AsyncGeneratorType)) call_stack.append(FrameCallGraphEntry(f)) - if is_async: - if f.f_back is not None and f.f_back.f_generator is None: + if _is_async_frame(f): + if f.f_back is not None and _is_sync_frame(f.f_back): # We've reached the bottom of the coroutine stack, which # must be the Task that runs it. break @@ -221,36 +240,12 @@ def add_line(line: str) -> None: ) for ste in st.call_stack: f = ste.frame - - if f.f_generator is None: - f = ste.frame - add_line( - f' | File {f.f_code.co_filename!r},' - f' line {f.f_lineno}, in' - f' {f.f_code.co_qualname}()' - ) - else: - c = f.f_generator - - try: - f = c.cr_frame - code = c.cr_code - tag = 'async' - except AttributeError: - try: - f = c.ag_frame - code = c.ag_code - tag = 'async generator' - except AttributeError: - f = c.gi_frame - code = c.gi_code - tag = 'generator' - - add_line( - f' | File {f.f_code.co_filename!r},' - f' line {f.f_lineno}, in' - f' {tag} {code.co_qualname}()' - ) + tag = _FRAME_TAGS.get(f.f_code.co_flags & _CO_GEN_FLAGS, '') + add_line( + f' | File {f.f_code.co_filename!r},' + f' line {f.f_lineno}, in' + f' {tag}{f.f_code.co_qualname}()' + ) if st.awaited_by: add_line( diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py index e438e5915b2a852..e2164a093f06b29 100644 --- a/Lib/test/test_asyncio/test_graph.py +++ b/Lib/test/test_asyncio/test_graph.py @@ -1,4 +1,5 @@ import asyncio +import inspect import io import sys import types @@ -20,19 +21,23 @@ def walk(s): if isinstance(s.future, asyncio.Task) else 'F' ] + def tag(frame): + # A coroutine awaited right after it is called may run + # without a coroutine object until it first suspends, so + # classify frames by their code rather than f_generator. + flags = frame.f_code.co_flags + if flags & inspect.CO_COROUTINE: + return "a" + if flags & inspect.CO_ASYNC_GENERATOR: + return "ag" + if flags & inspect.CO_GENERATOR: + return "g" + return "s" + ret.append( [ - ( - f"s {entry.frame.f_code.co_name}" - if entry.frame.f_generator is None else - ( - f"a {entry.frame.f_generator.cr_code.co_name}" - if hasattr(entry.frame.f_generator, 'cr_code') else - f"ag {entry.frame.f_generator.ag_code.co_name}" - if hasattr(entry.frame.f_generator, 'ag_code') else - f"g {entry.frame.f_generator.gi_code.co_name}" - ) - ) for entry in s.call_stack + f"{tag(entry.frame)} {entry.frame.f_code.co_name}" + for entry in s.call_stack # Skip the pure Python Future.__await__ generator frame if entry.frame.f_code is not future_await_code ] @@ -893,6 +898,26 @@ async def main(): await main() self.assertEqual(stack[:3], ['gen', 'middle', 'main']) + async def test_capture_call_graph_through_generator_based_coroutine(self): + # A native coroutine driven by a generator-based coroutine + # (yield from) keeps the frames of its callers. + stack = None + + async def native(): + nonlocal stack + graph = asyncio.capture_call_graph() + stack = [entry.frame.f_code.co_name for entry in graph.call_stack] + + @types.coroutine + def legacy(): + return (yield from native()) + + async def main(): + await legacy() + + await main() + self.assertEqual(stack[:3], ['native', 'legacy', 'main']) + @unittest.skipIf( not hasattr(asyncio.futures, "_c_future_add_to_awaited_by"), diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index c5900da09501b36..1aba48a0080823c 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -1,6 +1,8 @@ import ast import contextlib import copy +import builtins +import gc import inspect import pickle import sys @@ -2594,6 +2596,508 @@ def test_unawaited_warning_during_shutdown(self): assert_python_ok("-c", code) +@support.cpython_only +class LazyCoroutineTest(unittest.TestCase): + """A coroutine that is awaited right away (``await f()``) runs in + the calling thread's frame and only gets a coroutine object when + it first suspends. This must not be observable, except that no + coroutine object exists before the first suspension.""" + + @staticmethod + def coroutines_for(code): + return [o for o in gc.get_objects() + if type(o) is types.CoroutineType and o.cr_code is code] + + def test_no_object_before_first_suspension(self): + seen = [] + + async def inner(): + seen.append(len(self.coroutines_for(inner.__code__))) + await AsyncYield(1) + seen.append(len(self.coroutines_for(inner.__code__))) + return 42 + + async def outer(): + return await inner() + + self.assertEqual(run_async(outer()), ([1], 42)) + self.assertEqual(seen, [0, 1]) + + def test_return_without_suspending(self): + async def inner(x): + return x + 1 + + async def outer(): + total = 0 + for i in range(100): + total += await inner(i) + return total + + self.assertEqual(run_async(outer()), ([], sum(range(1, 101)))) + + def test_nested_chain(self): + async def chain(n): + if n == 0: + await AsyncYield("bottom") + return 0 + return 1 + await chain(n - 1) + + self.assertEqual(run_async(chain(20)), (["bottom"], 20)) + + def test_exception_before_first_suspension(self): + async def inner(): + raise ValueError("boom") + + async def outer(): + try: + await inner() + except ValueError as exc: + tb = exc.__traceback__ + names = [] + while tb is not None: + names.append(tb.tb_frame.f_code.co_name) + tb = tb.tb_next + return names + + self.assertEqual(run_async(outer()), ([], ["outer", "inner"])) + + def test_stopiteration_before_first_suspension(self): + async def inner(): + raise StopIteration + + async def outer(): + await inner() + + with self.assertRaisesRegex(RuntimeError, + "coroutine raised StopIteration"): + run_async(outer()) + + def test_frame_object(self): + async def inner(): + x = 1 + frame = sys._getframe() + self.assertIsNone(frame.f_generator) + self.assertEqual(frame.f_locals["x"], 1) + self.assertEqual(frame.f_back.f_code.co_name, "outer") + await AsyncYield(1) + coro = frame.f_generator + self.assertIsInstance(coro, types.CoroutineType) + self.assertIs(coro.cr_frame, frame) + self.assertIs(coro.cr_code, inner.__code__) + self.assertEqual(frame.f_locals["x"], 1) + self.assertEqual(frame.f_back.f_code.co_name, "outer") + x = 2 + return frame + + async def outer(): + return await inner() + + _, frame = run_async(outer()) + self.assertIsNone(frame.f_generator) # finished + self.assertEqual(frame.f_locals["x"], 2) + + def test_exc_info_suspend_inside_handler(self): + async def inner(): + try: + raise KeyError + except KeyError: + await AsyncYield(1) + self.assertIsInstance(sys.exc_info()[1], KeyError) + await AsyncYield(2) + self.assertIsInstance(sys.exc_info()[1], KeyError) + # The caller's context is visible after the handler. + self.assertIsInstance(sys.exc_info()[1], TypeError) + await AsyncYield(3) + self.assertIsInstance(sys.exc_info()[1], TypeError) + return "ok" + + async def outer(): + try: + raise TypeError + except TypeError: + result = await inner() + self.assertIsInstance(sys.exc_info()[1], TypeError) + self.assertIsNone(sys.exc_info()[1]) + return result + + self.assertEqual(run_async(outer()), ([1, 2, 3], "ok")) + + def test_exc_info_handler_without_suspending(self): + async def inner(): + try: + raise KeyError + except KeyError: + self.assertIsInstance(sys.exc_info()[1], KeyError) + self.assertIsInstance(sys.exc_info()[1], TypeError) + return "ok" + + async def outer(): + try: + raise TypeError + except TypeError: + result = await inner() + self.assertIsInstance(sys.exc_info()[1], TypeError) + self.assertIsNone(sys.exc_info()[1]) + return result + + self.assertEqual(run_async(outer()), ([], "ok")) + + def test_throw_into_materialized_coroutine(self): + async def inner(): + try: + await AsyncYield(1) + except ZeroDivisionError: + return "caught" + return "not caught" + + async def outer(): + return await inner() + + coro = outer() + self.assertEqual(coro.send(None), 1) + with self.assertRaises(StopIteration) as cm: + coro.throw(ZeroDivisionError) + self.assertEqual(cm.exception.value, "caught") + + def test_close_materialized_coroutine(self): + closed = [] + + async def inner(): + try: + await AsyncYield(1) + finally: + closed.append(True) + + async def outer(): + await inner() + + coro = outer() + self.assertEqual(coro.send(None), 1) + coro.close() + self.assertEqual(closed, [True]) + + def test_async_with(self): + log = [] + + class CM: + async def __aenter__(self): + log.append("enter") + return self + + async def __aexit__(self, *args): + log.append("exit") + return False + + async def outer(): + async with CM() as cm: + log.append(type(cm).__name__) + await AsyncYield(1) + return "done" + + self.assertEqual(run_async(outer()), ([1], "done")) + self.assertEqual(log, ["enter", "CM", "exit"]) + + def test_async_comprehension(self): + async def inner(x): + return x * 2 + + async def outer(): + return [await inner(x) for x in range(5)] + + self.assertEqual(run_async(outer()), ([], [0, 2, 4, 6, 8])) + + def test_origin_tracking(self): + orig_depth = sys.get_coroutine_origin_tracking_depth() + + async def inner(): + await AsyncYield(1) + return sys._getframe().f_generator.cr_origin + + async def outer(): + coro = inner() + eager = (await coro)[0] + lazy = (await inner())[0] + return eager, lazy + + sys.set_coroutine_origin_tracking_depth(2) + try: + _, (eager, lazy) = run_async(outer()) + finally: + sys.set_coroutine_origin_tracking_depth(orig_depth) + # Origin tracking disables the lazy path: the origin is the + # call, not the await, and is captured at creation. + self.assertEqual(eager[0], __file__) + self.assertEqual(eager[2], "outer") + self.assertEqual(lazy[0], __file__) + self.assertEqual(lazy[2], "outer") + self.assertEqual(lazy[1], eager[1] + 2) + + def test_origin_tracking_multiline_await(self): + orig_depth = sys.get_coroutine_origin_tracking_depth() + + async def inner(): + await AsyncYield(1) + return sys._getframe().f_generator.cr_origin + + async def outer(): + origin = await ( + inner() + ) + return origin[0][1] - outer.__code__.co_firstlineno + + sys.set_coroutine_origin_tracking_depth(1) + try: + _, line = run_async(outer()) + finally: + sys.set_coroutine_origin_tracking_depth(orig_depth) + # The line of the call, not of the await. + self.assertEqual(line, 2) + + def test_settrace_events(self): + async def inner(): + await AsyncYield(1) + return 1 + + async def eager(): + coro = inner() + return await coro + + async def lazy(): + return await inner() + + def trace(coro_fn): + events = [] + + def tracer(frame, event, arg): + if frame.f_code is inner.__code__: + events.append((event, frame.f_lineno - frame.f_code.co_firstlineno)) + return tracer + + sys.settrace(tracer) + try: + result = run_async(coro_fn()) + finally: + sys.settrace(None) + return events, result + + self.assertEqual(trace(lazy), trace(eager)) + + def test_monitoring_events(self): + async def inner(): + await AsyncYield(1) + return 1 + + async def eager(): + coro = inner() + return await coro + + async def lazy(): + return await inner() + + E = sys.monitoring.events + # No LINE events: they instrument the caller's SEND, which + # makes the await take the regular path. + names = {E.PY_START: "start", E.PY_RESUME: "resume", + E.PY_RETURN: "return", E.PY_YIELD: "yield", + E.STOP_ITERATION: "stop_iteration"} + codes = {inner.__code__: "inner", eager.__code__: "outer", + lazy.__code__: "outer"} + + def monitor(coro_fn): + events = [] + tool = sys.monitoring.DEBUGGER_ID + sys.monitoring.use_tool_id(tool, "test") + try: + for event, name in names.items(): + def callback(code, offset, *args, name=name): + if code in codes: + events.append( + (codes[code], name, type(args[-1]) if args else None)) + sys.monitoring.register_callback(tool, event, callback) + sys.monitoring.set_events(tool, sum(names)) + result = run_async(coro_fn()) + finally: + sys.monitoring.set_events(tool, 0) + sys.monitoring.free_tool_id(tool) + return events, result + + self.assertEqual(monitor(lazy), monitor(eager)) + self.assertIn(("outer", "stop_iteration", StopIteration), + monitor(lazy)[0]) + + def test_stop_iteration_event_without_suspending(self): + async def inner(): + return 42 + + async def outer(): + return await inner() + + E = sys.monitoring.events + events = [] + tool = sys.monitoring.DEBUGGER_ID + sys.monitoring.use_tool_id(tool, "test") + try: + def callback(code, offset, exc): + if code is outer.__code__: + events.append(exc.value) + sys.monitoring.register_callback(tool, E.STOP_ITERATION, callback) + sys.monitoring.set_events(tool, E.STOP_ITERATION) + self.assertEqual(run_async(outer()), ([], 42)) + finally: + sys.monitoring.set_events(tool, 0) + sys.monitoring.free_tool_id(tool) + self.assertEqual(events, [42]) + + def test_exc_state_after_suspending_inside_handler(self): + # The coroutine's own exception state must not keep the + # caller's exception once its handler is done. + async def inner(): + try: + raise KeyError + except KeyError: + await AsyncYield(1) + await AsyncYield(2) + return sys.exc_info()[1] + + async def outer(): + try: + raise TypeError + except TypeError: + return await inner() + + coro = outer() + coro.send(None) + inner_coro = coro.cr_await + inner_coro.send(None) + self.assertNotIn(TypeError, + map(type, gc.get_referents(inner_coro))) + # Resumed directly, not through outer: no handled exception. + with self.assertRaises(StopIteration) as cm: + inner_coro.send(None) + self.assertIsNone(cm.exception.value) + coro.close() + + def test_throw_into_coroutine_suspended_inside_handler(self): + async def inner(): + try: + raise KeyError + except KeyError: + try: + await AsyncYield(1) + except ZeroDivisionError: + pass + return sys.exception() + + async def outer(): + try: + raise TypeError + except TypeError: + result = await inner() + self.assertIsInstance(sys.exception(), TypeError) + return result + + coro = outer() + self.assertEqual(coro.send(None), 1) + with self.assertRaises(StopIteration) as cm: + coro.throw(ZeroDivisionError) + self.assertIsNone(cm.exception.value) + + def test_import_hook_locals(self): + seen = [] + orig_import = builtins.__import__ + + def hook(name, globals=None, locals=None, fromlist=(), level=0): + seen.append(locals) + return orig_import(name, globals, locals, fromlist, level) + + async def inner(): + import sys as _sys + return _sys + + async def outer(): + try: + raise TypeError + except TypeError: + return await inner() + + with support.swap_attr(builtins, "__import__", hook): + self.assertEqual(run_async(outer()), ([], sys)) + self.assertEqual(seen, [None]) + + def test_traceback_frame_keeps_no_caller_exception(self): + async def inner(): + raise ValueError + + async def outer(): + try: + raise TypeError + except TypeError: + try: + await inner() + except ValueError as exc: + return exc.__traceback__.tb_next.tb_frame + + _, frame = run_async(outer()) + self.assertIs(frame.f_code, inner.__code__) + self.assertNotIn(TypeError, map(type, gc.get_referents(frame))) + + def test_code_replaced_before_first_suspension(self): + # The coroutine object is created at the first suspension, so it + # must be sized for the code the frame runs, not func.__code__. + async def small(): + big, small # same free variables as big() + await AsyncYield(1) + + async def big(): + a = b = c = d = e = f = g = h = i = j = k = m = n = o = p = 1 + big.__code__ = small.__code__ + await AsyncYield(1) + return (a, b, c, d, e, f, g, h, i, j, k, m, n, o, p) + + async def outer(): + return await big() + + self.assertEqual(run_async(outer()), ([1], (1,) * 15)) + + def test_finished_frame_has_no_f_back(self): + async def inner(): + return sys._getframe() + + async def outer(): + return await inner() + + _, frame = run_async(outer()) + self.assertIs(frame.f_code, inner.__code__) + self.assertIsNone(frame.f_back) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_memory_error_at_first_suspension(self): + @types.coroutine + def suspend(): + yield 1 + + awaitable = suspend() + + async def inner(): + _testcapi.set_nomemory(0, 1) + try: + await awaitable + finally: + _testcapi.remove_mem_hooks() + return "resumed" + + async def outer(): + return await inner() + + coro = outer() + try: + with self.assertRaises(MemoryError): + coro.send(None) + finally: + _testcapi.remove_mem_hooks() + awaitable.close() + + @support.cpython_only @unittest.skipIf(_testcapi is None, "requires _testcapi") class CAPITest(unittest.TestCase): diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 709fa374704f1f5..2c3972c7394d01b 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -615,8 +615,6 @@ async def _asyncwith(c): LOAD_SPECIAL 2 (__aenter__) CALL 0 GET_AWAITABLE 1 - PUSH_NULL - LOAD_COMMON_CONSTANT 7 (None) L2: SEND 4 (to L5) L3: YIELD_VALUE 1 L4: RESUME 3 @@ -632,8 +630,6 @@ async def _asyncwith(c): LOAD_COMMON_CONSTANT 7 (None) CALL 3 GET_AWAITABLE 2 - PUSH_NULL - LOAD_COMMON_CONSTANT 7 (None) L8: SEND 4 (to L11) L9: YIELD_VALUE 1 L10: RESUME 3 @@ -647,14 +643,12 @@ async def _asyncwith(c): RETURN_VALUE %4d L12: CLEANUP_THROW - L13: JUMP_BACKWARD_NO_INTERRUPT 28 (to L5) + L13: JUMP_BACKWARD_NO_INTERRUPT 26 (to L5) L14: CLEANUP_THROW L15: JUMP_BACKWARD_NO_INTERRUPT 10 (to L11) L16: PUSH_EXC_INFO WITH_EXCEPT_START GET_AWAITABLE 2 - PUSH_NULL - LOAD_COMMON_CONSTANT 7 (None) L17: SEND 5 (to L21) L18: YIELD_VALUE 1 L19: RESUME 3 diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 5f7c15f93a8332a..f2570f4e220ae17 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1630,8 +1630,15 @@ def run_async(coro): except StopIteration: break + class Suspend: + def __await__(self): + yield + class CM: async def __aenter__(self): + # A coroutine that is awaited right away and never + # suspends doesn't execute the caller's SEND at all. + await Suspend() return self async def __aexit__(self, *exc): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-27-12-00-00.gh-issue-161170.lazyCo.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-27-12-00-00.gh-issue-161170.lazyCo.rst new file mode 100644 index 000000000000000..e7ba18578dfe11d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-27-12-00-00.gh-issue-161170.lazyCo.rst @@ -0,0 +1,3 @@ +Speed up ``await`` of a coroutine that completes without suspending by about +40% by running it without creating a coroutine object. Until such a coroutine +first suspends, :attr:`frame.f_generator` is ``None``. diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 8e7ba1f69553dda..e094e3c7ca9459b 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -5197,9 +5197,6 @@ null_in = stack_pointer[-3]; sub_iter = stack_pointer[-4]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - #if !_Py_TAIL_CALL_INTERP - assert(throwflag); - #endif assert(exc_value && PyExceptionInstance_Check(exc_value)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -6824,6 +6821,8 @@ INSTRUCTION_STATS(GET_AWAITABLE); _PyStackRef iterable; _PyStackRef iter; + _PyStackRef null; + _PyStackRef none; iterable = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -6839,8 +6838,12 @@ JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; + none = PyStackRef_None; stack_pointer[0] = iter; - stack_pointer += 1; + stack_pointer[1] = null; + stack_pointer[2] = none; + stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7650,7 +7653,9 @@ index_or_null = stack_pointer[-2]; receiver = stack_pointer[-3]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o) || + Py_IsNone(receiver_o)) + { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); @@ -8315,8 +8320,30 @@ { retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->owner == FRAME_OWNED_BY_THREAD; + if (lazy) { + assert(frame->lazy_coro); + stack_pointer[-1] = retval; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + _PyFrame_StackPointerInvalidate(frame); + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; @@ -8331,6 +8358,9 @@ _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -8346,6 +8376,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNone(stack_pointer[-2])); + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-2] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + } value = temp; LLTRACE_RESUME_FRAME(); } @@ -11442,32 +11477,68 @@ _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { - JUMP_TO_LABEL(error); + _Py_CODEUNIT *send = NULL; + #if TIER_ONE + if (_PyFrame_GetCode(frame)->co_flags & CO_COROUTINE) { + send = _PyEval_LazyCoroSendInstr(tstate, frame); + } + #endif + _PyErr_StackItem *exc_info = NULL; + if (send != NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); + _PyFrame_StackPointerInvalidate(frame); + if (exc_info == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_Clear(tstate); + _PyFrame_StackPointerInvalidate(frame); + send = NULL; + } + } + if (send != NULL) { + frame->lazy_coro = 1; + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + _PyInterpreterFrame *prev = frame->previous; + _PyFrame_StackPush(prev, PyStackRef_None); + _PyFrame_StackPush(prev, PyStackRef_NULL); + prev->instr_ptr = send; + assert(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg <= UINT16_MAX); + prev->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg); + res = PyStackRef_None; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + assert(STACK_LEVEL() <= 2); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); } - assert(STACK_LEVEL() <= 2); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -13583,8 +13654,30 @@ { retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->owner == FRAME_OWNED_BY_THREAD; + if (lazy) { + assert(frame->lazy_coro); + stack_pointer[-1] = retval; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + _PyFrame_StackPointerInvalidate(frame); + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; @@ -13599,6 +13692,9 @@ _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -13614,6 +13710,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNone(stack_pointer[-2])); + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-2] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + } value = temp; LLTRACE_RESUME_FRAME(); } diff --git a/Objects/genobject.c b/Objects/genobject.c index 1c95a569af8b9b5..6be35bcbcaf86b4 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1104,9 +1104,8 @@ PyTypeObject PyGen_Type = { }; static PyObject * -make_gen(PyTypeObject *type, PyFunctionObject *func) +make_gen(PyTypeObject *type, PyFunctionObject *func, PyCodeObject *code) { - PyCodeObject *code = (PyCodeObject *)func->func_code; int slots = _PyFrame_NumSlotsForCodeObject(code); PyGenObject *gen = PyObject_GC_NewVar(PyGenObject, type, slots); if (gen == NULL) { @@ -1127,18 +1126,44 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) return (PyObject *)gen; } +static PyObject * +make_coro(PyFunctionObject *func, PyCodeObject *code, + _PyInterpreterFrame *caller) +{ + PyObject *coro = make_gen(&PyCoro_Type, func, code); + if (!coro) { + return NULL; + } + PyThreadState *tstate = _PyThreadState_GET(); + int origin_depth = tstate->coroutine_origin_tracking_depth; + + if (origin_depth == 0) { + ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL; + } else { + _PyInterpreterFrame *frame = _PyFrame_GetFirstComplete(caller); + PyObject *cr_origin = _PyCoro_ComputeOrigin(origin_depth, frame); + ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin; + if (!cr_origin) { + Py_DECREF(coro); + return NULL; + } + } + return coro; +} + PyObject * _Py_MakeCoro(PyFunctionObject *func) { - int coro_flags = ((PyCodeObject *)func->func_code)->co_flags & + PyCodeObject *code = (PyCodeObject *)func->func_code; + int coro_flags = code->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR); assert(coro_flags); if (coro_flags == CO_GENERATOR) { - return make_gen(&PyGen_Type, func); + return make_gen(&PyGen_Type, func, code); } if (coro_flags == CO_ASYNC_GENERATOR) { PyAsyncGenObject *ag; - ag = (PyAsyncGenObject *)make_gen(&PyAsyncGen_Type, func); + ag = (PyAsyncGenObject *)make_gen(&PyAsyncGen_Type, func, code); if (ag == NULL) { return NULL; } @@ -1150,28 +1175,32 @@ _Py_MakeCoro(PyFunctionObject *func) } assert (coro_flags == CO_COROUTINE); - PyObject *coro = make_gen(&PyCoro_Type, func); - if (!coro) { - return NULL; - } PyThreadState *tstate = _PyThreadState_GET(); - int origin_depth = tstate->coroutine_origin_tracking_depth; - - if (origin_depth == 0) { - ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL; - } else { - _PyInterpreterFrame *frame = tstate->current_frame; - assert(frame); - assert(_PyFrame_IsIncomplete(frame)); - frame = _PyFrame_GetFirstComplete(frame->previous); - PyObject *cr_origin = _PyCoro_ComputeOrigin(origin_depth, frame); - ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin; - if (!cr_origin) { - Py_DECREF(coro); - return NULL; - } - } - return coro; + _PyInterpreterFrame *frame = tstate->current_frame; + assert(frame); + assert(_PyFrame_IsIncomplete(frame)); + return make_coro(func, code, frame->previous); +} + +/* Create the coroutine object for a coroutine whose body has been + * executing directly in `frame`, a frame owned by the thread, because + * its caller awaited the result immediately (see RETURN_GENERATOR in + * bytecodes.c). Called when the body suspends for the first time. + * The frame itself is moved into the coroutine by the caller. + * The body may have replaced func.__code__ in the meantime, so the + * coroutine is sized for the code the frame is actually running. */ +PyObject * +_PyCoro_MaterializeLazy(_PyInterpreterFrame *frame) +{ + assert(frame->owner == FRAME_OWNED_BY_THREAD); + assert(frame->lazy_coro); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = + (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyCodeObject *code = _PyFrame_GetCode(frame); + assert((code->co_flags & + (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) == CO_COROUTINE); + return make_coro(func, code, frame->previous); } static PyObject * diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 42ad83daa1ea461..32a0df0c6cc1fa6 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -437,7 +437,11 @@ dummy_func( tier1 inst(INSTRUMENTED_END_SEND, (receiver, index_or_null, value -- val)) { PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + /* None is the placeholder of a lazily started coroutine that + * returned before it was created (see RETURN_GENERATOR). */ + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o) || + Py_IsNone(receiver_o)) + { int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); if (err) { ERROR_NO_POP(); @@ -1649,11 +1653,17 @@ dummy_func( awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); } - inst(GET_AWAITABLE, (iterable -- iter)) { + // Also pushes the NULL and None that the following SEND loop + // starts with, so that `await` compiles to a single instruction + // between the awaited expression and SEND (RETURN_GENERATOR + // relies on this to recognize an immediately awaited call). + inst(GET_AWAITABLE, (iterable -- iter, null, none)) { PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); PyStackRef_CLOSE(iterable); ERROR_IF(iter_o == NULL); iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; + none = PyStackRef_None; } family(SEND, INLINE_CACHE_ENTRIES_SEND) = { @@ -1853,9 +1863,29 @@ dummy_func( // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. + // The one exception is a lazily created coroutine (see + // RETURN_GENERATOR) that cannot be allocated: the await is + // then treated as having raised MemoryError. assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->owner == FRAME_OWNED_BY_THREAD; + if (lazy) { + assert(frame->lazy_coro); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + if (gen == NULL) { + ERROR_NO_POP(); + } + /* Move the body's exception state (see RETURN_GENERATOR) + * into the coroutine, where SEND_GEN would have put it. */ + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; @@ -1868,6 +1898,9 @@ dummy_func( _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -1883,6 +1916,12 @@ dummy_func( #endif RELOAD_STACK(); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + /* Fill in the coroutine slot of the caller's SEND. */ + assert(PyStackRef_IsNone(stack_pointer[-2])); + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-2] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + } value = temp; LLTRACE_RESUME_FRAME(); } @@ -1962,9 +2001,9 @@ dummy_func( tier1 inst(CLEANUP_THROW, (sub_iter, null_in, last_sent_val, exc_value_st -- none, null_out, value)) { PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - #if !_Py_TAIL_CALL_INTERP - assert(throwflag); - #endif + /* Reached on throw()/close(), or when a lazily started + * coroutine could not be created at its first suspension + * (see _YIELD_VALUE). */ assert(exc_value && PyExceptionInstance_Check(exc_value)); int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); if (matches) { @@ -5806,25 +5845,68 @@ dummy_func( inst(RETURN_GENERATOR, (-- res)) { assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - ERROR_IF(gen == NULL); - assert(STACK_LEVEL() <= 2); - SAVE_STACK(); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - RELOAD_STACK(); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); + _Py_CODEUNIT *send = NULL; + #if TIER_ONE + if (_PyFrame_GetCode(frame)->co_flags & CO_COROUTINE) { + send = _PyEval_LazyCoroSendInstr(tstate, frame); + } + #endif + _PyErr_StackItem *exc_info = NULL; + if (send != NULL) { + /* The body gets its own exception state, as SEND_GEN + * would have given it. It lives on the data stack right + * after this frame; clear_thread_frame() and _YIELD_VALUE + * pop it. */ + exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); + if (exc_info == NULL) { + /* Out of memory: fall back to the regular path. */ + _PyErr_Clear(tstate); + send = NULL; + } + } + if (send != NULL) { + /* The caller is going to await the coroutine right away. + * Keep executing its body in this frame, and only create + * the coroutine object if the body suspends (see + * _YIELD_VALUE). Set up the caller's frame as if SEND_GEN + * had pushed this frame: `[receiver, NULL]` on its stack, + * with the coroutine slot filled in on materialization, + * and its instruction pointer at the SEND. */ + frame->lazy_coro = 1; + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + _PyInterpreterFrame *prev = frame->previous; + _PyFrame_StackPush(prev, PyStackRef_None); + _PyFrame_StackPush(prev, PyStackRef_NULL); + prev->instr_ptr = send; + assert(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg <= UINT16_MAX); + prev->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg); + /* The following POP_TOP discards this. */ + res = PyStackRef_None; + } + else { + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + ERROR_IF(gen == NULL); + assert(STACK_LEVEL() <= 2); + SAVE_STACK(); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + RELOAD_STACK(); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); + } } inst(BUILD_SLICE, (args[oparg] -- slice)) { diff --git a/Python/ceval.c b/Python/ceval.c index 223106065c603c4..69ac8405a4d22a5 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2063,10 +2063,19 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, return -1; } +static PyObject *pop_lazy_coro_exc_state(PyThreadState *, _PyInterpreterFrame *); + static void clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) { assert(frame->owner == FRAME_OWNED_BY_THREAD); + PyObject *lazy_exc = NULL; + if (frame->lazy_coro) { + lazy_exc = pop_lazy_coro_exc_state(tstate, frame); + // Like clear_gen_frame(): a frame object that outlives the + // coroutine must not keep the caller's frame alive via f_back. + frame->previous = NULL; + } // Make sure that this is, indeed, the top frame. We can't check this in // _PyThreadState_PopFrame, since f_code is already cleared at that point: assert((PyObject **)frame + _PyFrame_GetCode(frame)->co_framesize == @@ -2075,6 +2084,7 @@ clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) _PyFrame_ClearExceptCode(frame); PyStackRef_CLEAR(frame->f_executable); _PyThreadState_PopFrame(tstate, frame); + Py_XDECREF(lazy_exc); } static void @@ -3750,6 +3760,94 @@ _PyEval_LoadGlobalStackRef(PyObject *globals, PyObject *builtins, PyObject *name } } +/* RETURN_GENERATOR support for coroutines that are awaited immediately. + * + * If the caller of the coroutine frame `frame` is a Python frame whose next + * instruction is GET_AWAITABLE, i.e. the compiler emitted the call as an + * `await` operand (GET_AWAITABLE is always followed by `SEND; ...; END_SEND`), + * return the SEND instruction, otherwise NULL. Instrumented, + * EXTENDED_ARG-prefixed or executor-replaced instructions don't match, so + * the regular path is taken for them, and so is it while the JIT is + * recording a trace (tier 2 creates coroutines eagerly) and while + * coroutine origin tracking is on (the origin is captured at creation). */ +_Py_CODEUNIT * +_PyEval_LazyCoroSendInstr(PyThreadState *tstate, _PyInterpreterFrame *frame) +{ + int flags = _PyFrame_GetCode(frame)->co_flags; + if ((flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR | CO_OPTIMIZED)) + != (CO_COROUTINE | CO_OPTIMIZED)) + { + return NULL; + } + if (tstate->coroutine_origin_tracking_depth != 0) { + return NULL; + } +#ifdef _Py_TIER2 + _PyJitTracerState *tracer = ((_PyThreadStateImpl *)tstate)->jit_tracer_state; + if (tracer != NULL && tracer->is_tracing) { + return NULL; + } +#endif + _PyInterpreterFrame *prev = frame->previous; + if (prev->owner >= FRAME_OWNED_BY_INTERPRETER) { + /* Called from C: the caller can't be inspected. */ + return NULL; + } + _Py_CODEUNIT *instr = prev->instr_ptr + prev->return_offset; + if (instr[0].op.code != GET_AWAITABLE) { + return NULL; + } + _Py_CODEUNIT *send = &instr[1]; + if (_PyOpcode_Deopt[send->op.code] != SEND) { + /* Instrumented: the SEND must execute to emit its events. */ + return NULL; + } + _Py_CODEUNIT *end = send + 1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg; + if (end->op.code != END_SEND) { + /* INSTRUMENTED_END_SEND needs the coroutine object. */ + return NULL; + } + return send; +} + +/* Pop the exception state that RETURN_GENERATOR gave a lazily started + * coroutine body, restoring the caller's. Returns the body's exception, + * if any (a new reference). */ +static PyObject * +pop_lazy_coro_exc_state(PyThreadState *tstate, _PyInterpreterFrame *frame) +{ + assert(frame->lazy_coro); + _PyErr_StackItem *exc_info = tstate->exc_info; + tstate->exc_info = exc_info->previous_item; + PyObject *exc_value = exc_info->exc_value; + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + frame->lazy_coro = 0; + return exc_value; +} + +/* YIELD_VALUE support: move the frame of a lazily created coroutine + * from the thread's data stack into `gen`. Returns the new frame. */ +_PyInterpreterFrame * +_PyEval_MoveLazyCoroFrame(PyThreadState *tstate, PyGenObject *gen, + _PyInterpreterFrame *frame) +{ + assert(frame->owner == FRAME_OWNED_BY_THREAD); + assert(frame->lazy_coro); + assert((PyObject **)frame + _PyFrame_GetCode(frame)->co_framesize == + tstate->datastack_top); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + _PyFrame_Copy(frame, gen_frame); + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + PyFrameObject *f = gen_frame->frame_obj; + if (f != NULL) { + Py_BEGIN_CRITICAL_SECTION(f); + f->f_frame = gen_frame; + Py_END_CRITICAL_SECTION(); + } + _PyThreadState_PopFrame(tstate, frame); + return gen_frame; +} + PyObject * _PyEval_GetAwaitable(PyObject *iterable, int oparg) { diff --git a/Python/codegen.c b/Python/codegen.c index a975705576a167c..172d98a7d8b1100 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -506,7 +506,8 @@ codegen_add_yield_from(compiler *c, location loc, int await) USE_LABEL(c, send); ADDOP_JUMP(c, loc, SEND, exit); // Set up a virtual try/except to handle when StopIteration is raised during - // a close or throw call. The only way YIELD_VALUE raises if they do! + // a close or throw call. The only ways YIELD_VALUE raises are if they do, + // or if a lazily started coroutine (see RETURN_GENERATOR) can't be created. ADDOP_JUMP(c, loc, SETUP_FINALLY, fail); ADDOP_I(c, loc, YIELD_VALUE, 1); ADDOP(c, NO_LOCATION, POP_BLOCK); @@ -618,8 +619,6 @@ codegen_unwind_fblock(compiler *c, location *ploc, RETURN_IF_ERROR(codegen_call_exit_with_nones(c, *ploc)); if (info->fb_type == COMPILE_FBLOCK_ASYNC_WITH) { ADDOP_I(c, *ploc, GET_AWAITABLE, 2); - ADDOP(c, *ploc, PUSH_NULL); - ADDOP_LOAD_CONST(c, *ploc, Py_None); ADD_YIELD_FROM(c, *ploc, 1); } ADDOP(c, *ploc, POP_TOP); @@ -5167,8 +5166,6 @@ codegen_comprehension(compiler *c, expr_ty e, int type, if (is_async_comprehension && type != COMP_GENEXP) { ADDOP_I(c, loc, GET_AWAITABLE, 0); - ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); } @@ -5312,8 +5309,6 @@ codegen_async_with_inner(compiler *c, stmt_ty s, int pos) ADDOP_I(c, loc, LOAD_SPECIAL, SPECIAL___AENTER__); ADDOP_I(c, loc, CALL, 0); ADDOP_I(c, loc, GET_AWAITABLE, 1); - ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); ADDOP_JUMP(c, loc, SETUP_WITH, final); @@ -5349,8 +5344,6 @@ codegen_async_with_inner(compiler *c, stmt_ty s, int pos) */ RETURN_IF_ERROR(codegen_call_exit_with_nones(c, loc)); ADDOP_I(c, loc, GET_AWAITABLE, 2); - ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); ADDOP(c, loc, POP_TOP); @@ -5364,8 +5357,6 @@ codegen_async_with_inner(compiler *c, stmt_ty s, int pos) ADDOP(c, loc, PUSH_EXC_INFO); ADDOP(c, loc, WITH_EXCEPT_START); ADDOP_I(c, loc, GET_AWAITABLE, 2); - ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); RETURN_IF_ERROR(codegen_with_except_finish(c, cleanup)); @@ -5552,8 +5543,6 @@ codegen_visit_expr_impl(compiler *c, expr_ty e, bool result_is_unused) case Await_kind: VISIT(c, expr, e->v.Await.value); ADDOP_I(c, loc, GET_AWAITABLE, 0); - ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); break; case Compare_kind: diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index bc85ccd9ddb4fc1..88251d8861544dd 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -8906,11 +8906,13 @@ break; } - case _GET_AWAITABLE_r11: { + case _GET_AWAITABLE_r13: { CHECK_CURRENT_CACHED_VALUES(1); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iterable; _PyStackRef iter; + _PyStackRef null; + _PyStackRef none; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); iterable = _stack_item_0; @@ -8932,10 +8934,12 @@ JUMP_TO_ERROR(); } iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; + none = PyStackRef_None; + _tos_cache2 = none; + _tos_cache1 = null; _tos_cache0 = iter; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(3); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -9498,11 +9502,39 @@ oparg = CURRENT_OPARG(); retval = _stack_item_0; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->owner == FRAME_OWNED_BY_THREAD; + if (lazy) { + assert(frame->lazy_coro); + stack_pointer[0] = retval; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + _PyFrame_StackPointerInvalidate(frame); + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + stack_pointer += 1; + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); DTRACE_FUNCTION_RETURN(); @@ -9512,6 +9544,9 @@ _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -9527,6 +9562,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNone(stack_pointer[-2])); + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-2] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + } value = temp; LLTRACE_RESUME_FRAME(); _tos_cache0 = value; @@ -20720,33 +20760,69 @@ _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + _Py_CODEUNIT *send = NULL; + #if TIER_ONE + if (_PyFrame_GetCode(frame)->co_flags & CO_COROUTINE) { + send = _PyEval_LazyCoroSendInstr(tstate, frame); + } + #endif + _PyErr_StackItem *exc_info = NULL; + if (send != NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); + _PyFrame_StackPointerInvalidate(frame); + if (exc_info == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_Clear(tstate); + _PyFrame_StackPointerInvalidate(frame); + send = NULL; + } + } + if (send != NULL) { + frame->lazy_coro = 1; + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + _PyInterpreterFrame *prev = frame->previous; + _PyFrame_StackPush(prev, PyStackRef_None); + _PyFrame_StackPush(prev, PyStackRef_NULL); + prev->instr_ptr = send; + assert(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg <= UINT16_MAX); + prev->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg); + res = PyStackRef_None; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + assert(STACK_LEVEL() <= 2); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); } - assert(STACK_LEVEL() <= 2); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); _tos_cache0 = res; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ee9b64107bb9f24..a04a401138796eb 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5197,9 +5197,6 @@ null_in = stack_pointer[-3]; sub_iter = stack_pointer[-4]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - #if !_Py_TAIL_CALL_INTERP - assert(throwflag); - #endif assert(exc_value && PyExceptionInstance_Check(exc_value)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -6824,6 +6821,8 @@ INSTRUCTION_STATS(GET_AWAITABLE); _PyStackRef iterable; _PyStackRef iter; + _PyStackRef null; + _PyStackRef none; iterable = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -6839,8 +6838,12 @@ JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; + none = PyStackRef_None; stack_pointer[0] = iter; - stack_pointer += 1; + stack_pointer[1] = null; + stack_pointer[2] = none; + stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7650,7 +7653,9 @@ index_or_null = stack_pointer[-2]; receiver = stack_pointer[-3]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o) || + Py_IsNone(receiver_o)) + { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); @@ -8314,8 +8319,30 @@ { retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->owner == FRAME_OWNED_BY_THREAD; + if (lazy) { + assert(frame->lazy_coro); + stack_pointer[-1] = retval; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + _PyFrame_StackPointerInvalidate(frame); + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; @@ -8330,6 +8357,9 @@ _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -8345,6 +8375,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNone(stack_pointer[-2])); + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-2] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + } value = temp; LLTRACE_RESUME_FRAME(); } @@ -11439,32 +11474,68 @@ _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { - JUMP_TO_LABEL(error); + _Py_CODEUNIT *send = NULL; + #if TIER_ONE + if (_PyFrame_GetCode(frame)->co_flags & CO_COROUTINE) { + send = _PyEval_LazyCoroSendInstr(tstate, frame); + } + #endif + _PyErr_StackItem *exc_info = NULL; + if (send != NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); + _PyFrame_StackPointerInvalidate(frame); + if (exc_info == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_Clear(tstate); + _PyFrame_StackPointerInvalidate(frame); + send = NULL; + } + } + if (send != NULL) { + frame->lazy_coro = 1; + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + _PyInterpreterFrame *prev = frame->previous; + _PyFrame_StackPush(prev, PyStackRef_None); + _PyFrame_StackPush(prev, PyStackRef_NULL); + prev->instr_ptr = send; + assert(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg <= UINT16_MAX); + prev->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg); + res = PyStackRef_None; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + assert(STACK_LEVEL() <= 2); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); } - assert(STACK_LEVEL() <= 2); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -13580,8 +13651,30 @@ { retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->owner == FRAME_OWNED_BY_THREAD; + if (lazy) { + assert(frame->lazy_coro); + stack_pointer[-1] = retval; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + _PyFrame_StackPointerInvalidate(frame); + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; @@ -13596,6 +13689,9 @@ _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -13611,6 +13707,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNone(stack_pointer[-2])); + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-2] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + } value = temp; LLTRACE_RESUME_FRAME(); } diff --git a/Python/intrinsics.c b/Python/intrinsics.c index dfa81baa654e123..e3b0fd406a612a4 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -144,7 +144,11 @@ static PyObject * stopiteration_error(PyThreadState* tstate, PyObject *exc) { _PyInterpreterFrame *frame = tstate->current_frame; - assert(frame->owner == FRAME_OWNED_BY_GENERATOR); + /* A coroutine that is awaited immediately runs in a thread-owned + * frame until it first suspends (see RETURN_GENERATOR). */ + assert(frame->owner == FRAME_OWNED_BY_GENERATOR || + (frame->owner == FRAME_OWNED_BY_THREAD && + (_PyFrame_GetCode(frame)->co_flags & CO_COROUTINE))); assert(PyExceptionInstance_Check(exc)); const char *msg = NULL; if (PyErr_GivenExceptionMatches(exc, PyExc_StopIteration)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 9cd20c495db4418..86ed2c6d10503c1 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1888,8 +1888,17 @@ case _GET_AWAITABLE: { JitOptRef iter; + JitOptRef null; + JitOptRef none; iter = sym_new_not_null(ctx); + null = sym_new_null(ctx); + none = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(2); stack_pointer[-1] = iter; + stack_pointer[0] = null; + stack_pointer[1] = none; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index c3e0b0dbff21993..b6bb03554daa841 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -599,6 +599,7 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "_PyFrame_PushUnchecked", "_PyFrame_SetStackPointer", "_PyFrame_StackPush", + "_PyEval_LazyCoroSendInstr", "_PyFrame_StackAssertInvalid", "_PyFunction_SetVersion", "_PyGen_GetGeneratorFromFrame",