From bc054db9aa6d42c34acad0648054c582673f94ab Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 2 Dec 2025 14:48:58 -0500 Subject: [PATCH 1/9] gh-120321: Make gi_frame_state transitions atomic in FT build This makes generator frame state transitions atomic in the free threading build, which avoids segfaults when trying to execute a generator from multiple threads concurrently. There are still a few operations that aren't thread-safe and may crash if performed concurrently on the same generator/coroutine: * Accessing gi_yieldfrom/cr_await/ag_await * Accessing gi_frame/cr_frame/ag_frame * Async generator operations --- Include/cpython/pyatomic.h | 3 + Include/cpython/pyatomic_gcc.h | 4 + Include/cpython/pyatomic_msc.h | 13 + Include/cpython/pyatomic_std.h | 8 + Include/internal/pycore_frame.h | 14 - Include/internal/pycore_genobject.h | 38 +- .../internal/pycore_pyatomic_ft_wrappers.h | 9 + Include/internal/pycore_tstate.h | 4 + Include/internal/pycore_uop_ids.h | 1114 +++++++++-------- Include/internal/pycore_uop_metadata.h | 10 +- Modules/_remote_debugging/_remote_debugging.h | 2 +- Objects/genobject.c | 504 ++++---- Python/bytecodes.c | 21 +- Python/ceval.c | 17 +- Python/executor_cases.c.h | 93 +- Python/generated_cases.c.h | 28 +- Tools/cases_generator/analyzer.py | 1 + 17 files changed, 1032 insertions(+), 851 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 2a0c11e7b3ad66..8f3ee1368dbc0b 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -523,6 +523,9 @@ _Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value); static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value); +static inline void +_Py_atomic_store_int8_release(int8_t *obj, int8_t value); + static inline void _Py_atomic_store_int_release(int *obj, int value); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 1566b83b9f6a1b..c045213c898a03 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -572,6 +572,10 @@ static inline void _Py_atomic_store_int_release(int *obj, int value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_int8_release(int8_t *obj, int8_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index d155955df0cddf..8b9dd3eb0f8e16 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1066,6 +1066,19 @@ _Py_atomic_store_int_release(int *obj, int value) #endif } +static inline void +_Py_atomic_store_int8_release(int8_t *obj, int8_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(int8_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int8); + __stlr8((unsigned __int8 volatile *)obj, (unsigned __int8)value); +#else +# error "no implementation of _Py_atomic_store_int8_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 7176f667a4082c..cfc8dbefc63d09 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -1023,6 +1023,14 @@ _Py_atomic_store_int_release(int *obj, int value) memory_order_release); } +static inline void +_Py_atomic_store_int8_release(int8_t *obj, int8_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int8_t)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) { diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 8c410e9e208340..d930c9aeef15af 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -41,20 +41,6 @@ struct _frame { extern PyFrameObject* _PyFrame_New_NoTrack(PyCodeObject *code); -/* other API */ - -typedef enum _framestate { - FRAME_CREATED = -3, - FRAME_SUSPENDED = -2, - FRAME_SUSPENDED_YIELD_FROM = -1, - FRAME_EXECUTING = 0, - FRAME_COMPLETED = 1, - FRAME_CLEARED = 4 -} PyFrameState; - -#define FRAME_STATE_SUSPENDED(S) ((S) == FRAME_SUSPENDED || (S) == FRAME_SUSPENDED_YIELD_FROM) -#define FRAME_STATE_FINISHED(S) ((S) >= FRAME_COMPLETED) - #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index b08c8c52221f4b..d73bef50c5a9ba 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -10,8 +10,21 @@ extern "C" { #include "pycore_interpframe_structs.h" // _PyGenObject +#include // bool #include // offsetof() +#include "pycore_object.h" // _PyObject_IsUniquelyReferenced() +typedef enum _framestate { + FRAME_CREATED = -3, + FRAME_SUSPENDED = -2, + FRAME_SUSPENDED_YIELD_FROM = -1, + FRAME_EXECUTING = 0, + FRAME_COMPLETED = 1, + FRAME_CLEARED = 4 +} PyFrameState; + +#define FRAME_STATE_SUSPENDED(S) ((S) == FRAME_SUSPENDED || (S) == FRAME_SUSPENDED_YIELD_FROM) +#define FRAME_STATE_FINISHED(S) ((S) >= FRAME_COMPLETED) static inline PyGenObject *_PyGen_GetGeneratorFromFrame(_PyInterpreterFrame *frame) @@ -21,7 +34,30 @@ PyGenObject *_PyGen_GetGeneratorFromFrame(_PyInterpreterFrame *frame) return (PyGenObject *)(((char *)frame) - offset_in_gen); } -PyAPI_FUNC(PyObject *)_PyGen_yf(PyGenObject *); +// Mark the generator as executing. Returns true if the state was changed, +// false if it was already executing or finished. +static inline bool +_PyGen_SetExecuting(PyGenObject *gen) +{ +#ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { + int8_t frame_state = _Py_atomic_load_int8_relaxed(&gen->gi_frame_state); + while (frame_state < FRAME_EXECUTING) { + if (_Py_atomic_compare_exchange_int8(&gen->gi_frame_state, + &frame_state, + FRAME_EXECUTING)) { + return true; + } + } + } +#endif + if (gen->gi_frame_state < FRAME_EXECUTING) { + gen->gi_frame_state = FRAME_EXECUTING; + return true; + } + return false; +} + extern void _PyGen_Finalize(PyObject *self); // Export for '_asyncio' shared extension diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 2ae0185226f847..87683fa734af72 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -39,6 +39,8 @@ extern "C" { _Py_atomic_load_uint8(&value) #define FT_ATOMIC_STORE_UINT8(value, new_value) \ _Py_atomic_store_uint8(&value, new_value) +#define FT_ATOMIC_LOAD_INT8_RELAXED(value) \ + _Py_atomic_load_int8_relaxed(&value) #define FT_ATOMIC_LOAD_UINT8_RELAXED(value) \ _Py_atomic_load_uint8_relaxed(&value) #define FT_ATOMIC_LOAD_UINT16_RELAXED(value) \ @@ -53,6 +55,10 @@ extern "C" { _Py_atomic_store_ptr_release(&value, new_value) #define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) \ _Py_atomic_store_uintptr_release(&value, new_value) +#define FT_ATOMIC_STORE_INT8_RELAXED(value, new_value) \ + _Py_atomic_store_int8_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_INT8_RELEASE(value, new_value) \ + _Py_atomic_store_int8_release(&value, new_value) #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) \ _Py_atomic_store_ssize_relaxed(&value, new_value) #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) \ @@ -129,6 +135,7 @@ extern "C" { #define FT_ATOMIC_LOAD_PTR_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT8(value) value #define FT_ATOMIC_STORE_UINT8(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_INT8_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT8_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT16_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT32_RELAXED(value) value @@ -136,6 +143,8 @@ extern "C" { #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_INT8_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_INT8_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index c4f723ac8abbbe..a10d805e0d6676 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -113,6 +113,10 @@ typedef struct _PyThreadStateImpl { // When >1, code objects do not immortalize their non-string constants. int suppress_co_const_immortalization; + // Last known frame state for generators/coroutines in this thread + // Used by genobject.c + int8_t gen_last_frame_state; + #ifdef Py_STATS // per-thread stats, will be merged into interp->pystats_struct PyStats *pystats_struct; // allocated by _PyStats_ThreadInit() diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index ce1ad5a4c8a5f6..f22ffa49bf5ab1 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -553,562 +553,564 @@ extern "C" { #define _FORMAT_SIMPLE_r11 746 #define _FORMAT_WITH_SPEC_r21 747 #define _FOR_ITER_r23 748 -#define _FOR_ITER_GEN_FRAME_r23 749 -#define _FOR_ITER_TIER_TWO_r23 750 -#define _GET_AITER_r11 751 -#define _GET_ANEXT_r12 752 -#define _GET_AWAITABLE_r11 753 -#define _GET_ITER_r12 754 -#define _GET_LEN_r12 755 -#define _GET_YIELD_FROM_ITER_r11 756 -#define _GUARD_BINARY_OP_EXTEND_r22 757 -#define _GUARD_CALLABLE_ISINSTANCE_r03 758 -#define _GUARD_CALLABLE_ISINSTANCE_r13 759 -#define _GUARD_CALLABLE_ISINSTANCE_r23 760 -#define _GUARD_CALLABLE_ISINSTANCE_r33 761 -#define _GUARD_CALLABLE_LEN_r03 762 -#define _GUARD_CALLABLE_LEN_r13 763 -#define _GUARD_CALLABLE_LEN_r23 764 -#define _GUARD_CALLABLE_LEN_r33 765 -#define _GUARD_CALLABLE_LIST_APPEND_r03 766 -#define _GUARD_CALLABLE_LIST_APPEND_r13 767 -#define _GUARD_CALLABLE_LIST_APPEND_r23 768 -#define _GUARD_CALLABLE_LIST_APPEND_r33 769 -#define _GUARD_CALLABLE_STR_1_r03 770 -#define _GUARD_CALLABLE_STR_1_r13 771 -#define _GUARD_CALLABLE_STR_1_r23 772 -#define _GUARD_CALLABLE_STR_1_r33 773 -#define _GUARD_CALLABLE_TUPLE_1_r03 774 -#define _GUARD_CALLABLE_TUPLE_1_r13 775 -#define _GUARD_CALLABLE_TUPLE_1_r23 776 -#define _GUARD_CALLABLE_TUPLE_1_r33 777 -#define _GUARD_CALLABLE_TYPE_1_r03 778 -#define _GUARD_CALLABLE_TYPE_1_r13 779 -#define _GUARD_CALLABLE_TYPE_1_r23 780 -#define _GUARD_CALLABLE_TYPE_1_r33 781 -#define _GUARD_DORV_NO_DICT_r01 782 -#define _GUARD_DORV_NO_DICT_r11 783 -#define _GUARD_DORV_NO_DICT_r22 784 -#define _GUARD_DORV_NO_DICT_r33 785 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01 786 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11 787 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22 788 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 789 -#define _GUARD_GLOBALS_VERSION_r00 790 -#define _GUARD_GLOBALS_VERSION_r11 791 -#define _GUARD_GLOBALS_VERSION_r22 792 -#define _GUARD_GLOBALS_VERSION_r33 793 -#define _GUARD_IP_RETURN_GENERATOR_r00 794 -#define _GUARD_IP_RETURN_GENERATOR_r11 795 -#define _GUARD_IP_RETURN_GENERATOR_r22 796 -#define _GUARD_IP_RETURN_GENERATOR_r33 797 -#define _GUARD_IP_RETURN_VALUE_r00 798 -#define _GUARD_IP_RETURN_VALUE_r11 799 -#define _GUARD_IP_RETURN_VALUE_r22 800 -#define _GUARD_IP_RETURN_VALUE_r33 801 -#define _GUARD_IP_YIELD_VALUE_r00 802 -#define _GUARD_IP_YIELD_VALUE_r11 803 -#define _GUARD_IP_YIELD_VALUE_r22 804 -#define _GUARD_IP_YIELD_VALUE_r33 805 -#define _GUARD_IP__PUSH_FRAME_r00 806 -#define _GUARD_IP__PUSH_FRAME_r11 807 -#define _GUARD_IP__PUSH_FRAME_r22 808 -#define _GUARD_IP__PUSH_FRAME_r33 809 -#define _GUARD_IS_FALSE_POP_r00 810 -#define _GUARD_IS_FALSE_POP_r10 811 -#define _GUARD_IS_FALSE_POP_r21 812 -#define _GUARD_IS_FALSE_POP_r32 813 -#define _GUARD_IS_NONE_POP_r00 814 -#define _GUARD_IS_NONE_POP_r10 815 -#define _GUARD_IS_NONE_POP_r21 816 -#define _GUARD_IS_NONE_POP_r32 817 -#define _GUARD_IS_NOT_NONE_POP_r10 818 -#define _GUARD_IS_TRUE_POP_r00 819 -#define _GUARD_IS_TRUE_POP_r10 820 -#define _GUARD_IS_TRUE_POP_r21 821 -#define _GUARD_IS_TRUE_POP_r32 822 -#define _GUARD_KEYS_VERSION_r01 823 -#define _GUARD_KEYS_VERSION_r11 824 -#define _GUARD_KEYS_VERSION_r22 825 -#define _GUARD_KEYS_VERSION_r33 826 -#define _GUARD_NOS_DICT_r02 827 -#define _GUARD_NOS_DICT_r12 828 -#define _GUARD_NOS_DICT_r22 829 -#define _GUARD_NOS_DICT_r33 830 -#define _GUARD_NOS_FLOAT_r02 831 -#define _GUARD_NOS_FLOAT_r12 832 -#define _GUARD_NOS_FLOAT_r22 833 -#define _GUARD_NOS_FLOAT_r33 834 -#define _GUARD_NOS_INT_r02 835 -#define _GUARD_NOS_INT_r12 836 -#define _GUARD_NOS_INT_r22 837 -#define _GUARD_NOS_INT_r33 838 -#define _GUARD_NOS_LIST_r02 839 -#define _GUARD_NOS_LIST_r12 840 -#define _GUARD_NOS_LIST_r22 841 -#define _GUARD_NOS_LIST_r33 842 -#define _GUARD_NOS_NOT_NULL_r02 843 -#define _GUARD_NOS_NOT_NULL_r12 844 -#define _GUARD_NOS_NOT_NULL_r22 845 -#define _GUARD_NOS_NOT_NULL_r33 846 -#define _GUARD_NOS_NULL_r02 847 -#define _GUARD_NOS_NULL_r12 848 -#define _GUARD_NOS_NULL_r22 849 -#define _GUARD_NOS_NULL_r33 850 -#define _GUARD_NOS_OVERFLOWED_r02 851 -#define _GUARD_NOS_OVERFLOWED_r12 852 -#define _GUARD_NOS_OVERFLOWED_r22 853 -#define _GUARD_NOS_OVERFLOWED_r33 854 -#define _GUARD_NOS_TUPLE_r02 855 -#define _GUARD_NOS_TUPLE_r12 856 -#define _GUARD_NOS_TUPLE_r22 857 -#define _GUARD_NOS_TUPLE_r33 858 -#define _GUARD_NOS_UNICODE_r02 859 -#define _GUARD_NOS_UNICODE_r12 860 -#define _GUARD_NOS_UNICODE_r22 861 -#define _GUARD_NOS_UNICODE_r33 862 -#define _GUARD_NOT_EXHAUSTED_LIST_r02 863 -#define _GUARD_NOT_EXHAUSTED_LIST_r12 864 -#define _GUARD_NOT_EXHAUSTED_LIST_r22 865 -#define _GUARD_NOT_EXHAUSTED_LIST_r33 866 -#define _GUARD_NOT_EXHAUSTED_RANGE_r02 867 -#define _GUARD_NOT_EXHAUSTED_RANGE_r12 868 -#define _GUARD_NOT_EXHAUSTED_RANGE_r22 869 -#define _GUARD_NOT_EXHAUSTED_RANGE_r33 870 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r02 871 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r12 872 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r22 873 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r33 874 -#define _GUARD_THIRD_NULL_r03 875 -#define _GUARD_THIRD_NULL_r13 876 -#define _GUARD_THIRD_NULL_r23 877 -#define _GUARD_THIRD_NULL_r33 878 -#define _GUARD_TOS_ANY_SET_r01 879 -#define _GUARD_TOS_ANY_SET_r11 880 -#define _GUARD_TOS_ANY_SET_r22 881 -#define _GUARD_TOS_ANY_SET_r33 882 -#define _GUARD_TOS_DICT_r01 883 -#define _GUARD_TOS_DICT_r11 884 -#define _GUARD_TOS_DICT_r22 885 -#define _GUARD_TOS_DICT_r33 886 -#define _GUARD_TOS_FLOAT_r01 887 -#define _GUARD_TOS_FLOAT_r11 888 -#define _GUARD_TOS_FLOAT_r22 889 -#define _GUARD_TOS_FLOAT_r33 890 -#define _GUARD_TOS_INT_r01 891 -#define _GUARD_TOS_INT_r11 892 -#define _GUARD_TOS_INT_r22 893 -#define _GUARD_TOS_INT_r33 894 -#define _GUARD_TOS_LIST_r01 895 -#define _GUARD_TOS_LIST_r11 896 -#define _GUARD_TOS_LIST_r22 897 -#define _GUARD_TOS_LIST_r33 898 -#define _GUARD_TOS_OVERFLOWED_r01 899 -#define _GUARD_TOS_OVERFLOWED_r11 900 -#define _GUARD_TOS_OVERFLOWED_r22 901 -#define _GUARD_TOS_OVERFLOWED_r33 902 -#define _GUARD_TOS_SLICE_r01 903 -#define _GUARD_TOS_SLICE_r11 904 -#define _GUARD_TOS_SLICE_r22 905 -#define _GUARD_TOS_SLICE_r33 906 -#define _GUARD_TOS_TUPLE_r01 907 -#define _GUARD_TOS_TUPLE_r11 908 -#define _GUARD_TOS_TUPLE_r22 909 -#define _GUARD_TOS_TUPLE_r33 910 -#define _GUARD_TOS_UNICODE_r01 911 -#define _GUARD_TOS_UNICODE_r11 912 -#define _GUARD_TOS_UNICODE_r22 913 -#define _GUARD_TOS_UNICODE_r33 914 -#define _GUARD_TYPE_VERSION_r01 915 -#define _GUARD_TYPE_VERSION_r11 916 -#define _GUARD_TYPE_VERSION_r22 917 -#define _GUARD_TYPE_VERSION_r33 918 -#define _GUARD_TYPE_VERSION_AND_LOCK_r01 919 -#define _GUARD_TYPE_VERSION_AND_LOCK_r11 920 -#define _GUARD_TYPE_VERSION_AND_LOCK_r22 921 -#define _GUARD_TYPE_VERSION_AND_LOCK_r33 922 -#define _HANDLE_PENDING_AND_DEOPT_r00 923 -#define _HANDLE_PENDING_AND_DEOPT_r10 924 -#define _HANDLE_PENDING_AND_DEOPT_r20 925 -#define _HANDLE_PENDING_AND_DEOPT_r30 926 -#define _IMPORT_FROM_r12 927 -#define _IMPORT_NAME_r21 928 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 929 -#define _INIT_CALL_PY_EXACT_ARGS_r01 930 -#define _INIT_CALL_PY_EXACT_ARGS_0_r01 931 -#define _INIT_CALL_PY_EXACT_ARGS_1_r01 932 -#define _INIT_CALL_PY_EXACT_ARGS_2_r01 933 -#define _INIT_CALL_PY_EXACT_ARGS_3_r01 934 -#define _INIT_CALL_PY_EXACT_ARGS_4_r01 935 -#define _INSERT_NULL_r10 936 -#define _INSTRUMENTED_FOR_ITER_r23 937 -#define _INSTRUMENTED_INSTRUCTION_r00 938 -#define _INSTRUMENTED_JUMP_FORWARD_r00 939 -#define _INSTRUMENTED_JUMP_FORWARD_r11 940 -#define _INSTRUMENTED_JUMP_FORWARD_r22 941 -#define _INSTRUMENTED_JUMP_FORWARD_r33 942 -#define _INSTRUMENTED_LINE_r00 943 -#define _INSTRUMENTED_NOT_TAKEN_r00 944 -#define _INSTRUMENTED_NOT_TAKEN_r11 945 -#define _INSTRUMENTED_NOT_TAKEN_r22 946 -#define _INSTRUMENTED_NOT_TAKEN_r33 947 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r00 948 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 949 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 950 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 951 -#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 952 -#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 953 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r00 954 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 955 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 956 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 957 -#define _IS_NONE_r11 958 -#define _IS_OP_r21 959 -#define _ITER_CHECK_LIST_r02 960 -#define _ITER_CHECK_LIST_r12 961 -#define _ITER_CHECK_LIST_r22 962 -#define _ITER_CHECK_LIST_r33 963 -#define _ITER_CHECK_RANGE_r02 964 -#define _ITER_CHECK_RANGE_r12 965 -#define _ITER_CHECK_RANGE_r22 966 -#define _ITER_CHECK_RANGE_r33 967 -#define _ITER_CHECK_TUPLE_r02 968 -#define _ITER_CHECK_TUPLE_r12 969 -#define _ITER_CHECK_TUPLE_r22 970 -#define _ITER_CHECK_TUPLE_r33 971 -#define _ITER_JUMP_LIST_r02 972 -#define _ITER_JUMP_LIST_r12 973 -#define _ITER_JUMP_LIST_r22 974 -#define _ITER_JUMP_LIST_r33 975 -#define _ITER_JUMP_RANGE_r02 976 -#define _ITER_JUMP_RANGE_r12 977 -#define _ITER_JUMP_RANGE_r22 978 -#define _ITER_JUMP_RANGE_r33 979 -#define _ITER_JUMP_TUPLE_r02 980 -#define _ITER_JUMP_TUPLE_r12 981 -#define _ITER_JUMP_TUPLE_r22 982 -#define _ITER_JUMP_TUPLE_r33 983 -#define _ITER_NEXT_LIST_r23 984 -#define _ITER_NEXT_LIST_TIER_TWO_r23 985 -#define _ITER_NEXT_RANGE_r03 986 -#define _ITER_NEXT_RANGE_r13 987 -#define _ITER_NEXT_RANGE_r23 988 -#define _ITER_NEXT_TUPLE_r03 989 -#define _ITER_NEXT_TUPLE_r13 990 -#define _ITER_NEXT_TUPLE_r23 991 -#define _JUMP_BACKWARD_NO_INTERRUPT_r00 992 -#define _JUMP_BACKWARD_NO_INTERRUPT_r11 993 -#define _JUMP_BACKWARD_NO_INTERRUPT_r22 994 -#define _JUMP_BACKWARD_NO_INTERRUPT_r33 995 -#define _JUMP_TO_TOP_r00 996 -#define _LIST_APPEND_r10 997 -#define _LIST_EXTEND_r10 998 -#define _LOAD_ATTR_r10 999 -#define _LOAD_ATTR_CLASS_r11 1000 -#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_r11 1001 -#define _LOAD_ATTR_INSTANCE_VALUE_r11 1002 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r02 1003 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 1004 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 1005 -#define _LOAD_ATTR_METHOD_NO_DICT_r02 1006 -#define _LOAD_ATTR_METHOD_NO_DICT_r12 1007 -#define _LOAD_ATTR_METHOD_NO_DICT_r23 1008 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r02 1009 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 1010 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 1011 -#define _LOAD_ATTR_MODULE_r11 1012 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1013 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1014 -#define _LOAD_ATTR_PROPERTY_FRAME_r11 1015 -#define _LOAD_ATTR_SLOT_r11 1016 -#define _LOAD_ATTR_WITH_HINT_r11 1017 -#define _LOAD_BUILD_CLASS_r01 1018 -#define _LOAD_BYTECODE_r00 1019 -#define _LOAD_COMMON_CONSTANT_r01 1020 -#define _LOAD_COMMON_CONSTANT_r12 1021 -#define _LOAD_COMMON_CONSTANT_r23 1022 -#define _LOAD_CONST_r01 1023 -#define _LOAD_CONST_r12 1024 -#define _LOAD_CONST_r23 1025 -#define _LOAD_CONST_INLINE_r01 1026 -#define _LOAD_CONST_INLINE_r12 1027 -#define _LOAD_CONST_INLINE_r23 1028 -#define _LOAD_CONST_INLINE_BORROW_r01 1029 -#define _LOAD_CONST_INLINE_BORROW_r12 1030 -#define _LOAD_CONST_INLINE_BORROW_r23 1031 -#define _LOAD_CONST_UNDER_INLINE_r02 1032 -#define _LOAD_CONST_UNDER_INLINE_r12 1033 -#define _LOAD_CONST_UNDER_INLINE_r23 1034 -#define _LOAD_CONST_UNDER_INLINE_BORROW_r02 1035 -#define _LOAD_CONST_UNDER_INLINE_BORROW_r12 1036 -#define _LOAD_CONST_UNDER_INLINE_BORROW_r23 1037 -#define _LOAD_DEREF_r01 1038 -#define _LOAD_FAST_r01 1039 -#define _LOAD_FAST_r12 1040 -#define _LOAD_FAST_r23 1041 -#define _LOAD_FAST_0_r01 1042 -#define _LOAD_FAST_0_r12 1043 -#define _LOAD_FAST_0_r23 1044 -#define _LOAD_FAST_1_r01 1045 -#define _LOAD_FAST_1_r12 1046 -#define _LOAD_FAST_1_r23 1047 -#define _LOAD_FAST_2_r01 1048 -#define _LOAD_FAST_2_r12 1049 -#define _LOAD_FAST_2_r23 1050 -#define _LOAD_FAST_3_r01 1051 -#define _LOAD_FAST_3_r12 1052 -#define _LOAD_FAST_3_r23 1053 -#define _LOAD_FAST_4_r01 1054 -#define _LOAD_FAST_4_r12 1055 -#define _LOAD_FAST_4_r23 1056 -#define _LOAD_FAST_5_r01 1057 -#define _LOAD_FAST_5_r12 1058 -#define _LOAD_FAST_5_r23 1059 -#define _LOAD_FAST_6_r01 1060 -#define _LOAD_FAST_6_r12 1061 -#define _LOAD_FAST_6_r23 1062 -#define _LOAD_FAST_7_r01 1063 -#define _LOAD_FAST_7_r12 1064 -#define _LOAD_FAST_7_r23 1065 -#define _LOAD_FAST_AND_CLEAR_r01 1066 -#define _LOAD_FAST_AND_CLEAR_r12 1067 -#define _LOAD_FAST_AND_CLEAR_r23 1068 -#define _LOAD_FAST_BORROW_r01 1069 -#define _LOAD_FAST_BORROW_r12 1070 -#define _LOAD_FAST_BORROW_r23 1071 -#define _LOAD_FAST_BORROW_0_r01 1072 -#define _LOAD_FAST_BORROW_0_r12 1073 -#define _LOAD_FAST_BORROW_0_r23 1074 -#define _LOAD_FAST_BORROW_1_r01 1075 -#define _LOAD_FAST_BORROW_1_r12 1076 -#define _LOAD_FAST_BORROW_1_r23 1077 -#define _LOAD_FAST_BORROW_2_r01 1078 -#define _LOAD_FAST_BORROW_2_r12 1079 -#define _LOAD_FAST_BORROW_2_r23 1080 -#define _LOAD_FAST_BORROW_3_r01 1081 -#define _LOAD_FAST_BORROW_3_r12 1082 -#define _LOAD_FAST_BORROW_3_r23 1083 -#define _LOAD_FAST_BORROW_4_r01 1084 -#define _LOAD_FAST_BORROW_4_r12 1085 -#define _LOAD_FAST_BORROW_4_r23 1086 -#define _LOAD_FAST_BORROW_5_r01 1087 -#define _LOAD_FAST_BORROW_5_r12 1088 -#define _LOAD_FAST_BORROW_5_r23 1089 -#define _LOAD_FAST_BORROW_6_r01 1090 -#define _LOAD_FAST_BORROW_6_r12 1091 -#define _LOAD_FAST_BORROW_6_r23 1092 -#define _LOAD_FAST_BORROW_7_r01 1093 -#define _LOAD_FAST_BORROW_7_r12 1094 -#define _LOAD_FAST_BORROW_7_r23 1095 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1096 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1097 -#define _LOAD_FAST_CHECK_r01 1098 -#define _LOAD_FAST_LOAD_FAST_r02 1099 -#define _LOAD_FAST_LOAD_FAST_r13 1100 -#define _LOAD_FROM_DICT_OR_DEREF_r11 1101 -#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1102 -#define _LOAD_GLOBAL_r00 1103 -#define _LOAD_GLOBAL_BUILTINS_r01 1104 -#define _LOAD_GLOBAL_MODULE_r01 1105 -#define _LOAD_LOCALS_r01 1106 -#define _LOAD_NAME_r01 1107 -#define _LOAD_SMALL_INT_r01 1108 -#define _LOAD_SMALL_INT_r12 1109 -#define _LOAD_SMALL_INT_r23 1110 -#define _LOAD_SMALL_INT_0_r01 1111 -#define _LOAD_SMALL_INT_0_r12 1112 -#define _LOAD_SMALL_INT_0_r23 1113 -#define _LOAD_SMALL_INT_1_r01 1114 -#define _LOAD_SMALL_INT_1_r12 1115 -#define _LOAD_SMALL_INT_1_r23 1116 -#define _LOAD_SMALL_INT_2_r01 1117 -#define _LOAD_SMALL_INT_2_r12 1118 -#define _LOAD_SMALL_INT_2_r23 1119 -#define _LOAD_SMALL_INT_3_r01 1120 -#define _LOAD_SMALL_INT_3_r12 1121 -#define _LOAD_SMALL_INT_3_r23 1122 -#define _LOAD_SPECIAL_r00 1123 -#define _LOAD_SUPER_ATTR_ATTR_r31 1124 -#define _LOAD_SUPER_ATTR_METHOD_r32 1125 -#define _MAKE_CALLARGS_A_TUPLE_r33 1126 -#define _MAKE_CELL_r00 1127 -#define _MAKE_FUNCTION_r11 1128 -#define _MAKE_WARM_r00 1129 -#define _MAKE_WARM_r11 1130 -#define _MAKE_WARM_r22 1131 -#define _MAKE_WARM_r33 1132 -#define _MAP_ADD_r20 1133 -#define _MATCH_CLASS_r31 1134 -#define _MATCH_KEYS_r23 1135 -#define _MATCH_MAPPING_r02 1136 -#define _MATCH_MAPPING_r12 1137 -#define _MATCH_MAPPING_r23 1138 -#define _MATCH_SEQUENCE_r02 1139 -#define _MATCH_SEQUENCE_r12 1140 -#define _MATCH_SEQUENCE_r23 1141 -#define _MAYBE_EXPAND_METHOD_r00 1142 -#define _MAYBE_EXPAND_METHOD_KW_r11 1143 -#define _MONITOR_CALL_r00 1144 -#define _MONITOR_CALL_KW_r11 1145 -#define _MONITOR_JUMP_BACKWARD_r00 1146 -#define _MONITOR_JUMP_BACKWARD_r11 1147 -#define _MONITOR_JUMP_BACKWARD_r22 1148 -#define _MONITOR_JUMP_BACKWARD_r33 1149 -#define _MONITOR_RESUME_r00 1150 -#define _NOP_r00 1151 -#define _NOP_r11 1152 -#define _NOP_r22 1153 -#define _NOP_r33 1154 -#define _POP_CALL_r20 1155 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 1156 -#define _POP_CALL_ONE_r30 1157 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 1158 -#define _POP_CALL_TWO_r30 1159 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 1160 -#define _POP_EXCEPT_r10 1161 -#define _POP_ITER_r20 1162 -#define _POP_JUMP_IF_FALSE_r00 1163 -#define _POP_JUMP_IF_FALSE_r10 1164 -#define _POP_JUMP_IF_FALSE_r21 1165 -#define _POP_JUMP_IF_FALSE_r32 1166 -#define _POP_JUMP_IF_TRUE_r00 1167 -#define _POP_JUMP_IF_TRUE_r10 1168 -#define _POP_JUMP_IF_TRUE_r21 1169 -#define _POP_JUMP_IF_TRUE_r32 1170 -#define _POP_TOP_r10 1171 -#define _POP_TOP_FLOAT_r00 1172 -#define _POP_TOP_FLOAT_r10 1173 -#define _POP_TOP_FLOAT_r21 1174 -#define _POP_TOP_FLOAT_r32 1175 -#define _POP_TOP_INT_r00 1176 -#define _POP_TOP_INT_r10 1177 -#define _POP_TOP_INT_r21 1178 -#define _POP_TOP_INT_r32 1179 -#define _POP_TOP_LOAD_CONST_INLINE_r11 1180 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 1181 -#define _POP_TOP_NOP_r00 1182 -#define _POP_TOP_NOP_r10 1183 -#define _POP_TOP_NOP_r21 1184 -#define _POP_TOP_NOP_r32 1185 -#define _POP_TOP_UNICODE_r00 1186 -#define _POP_TOP_UNICODE_r10 1187 -#define _POP_TOP_UNICODE_r21 1188 -#define _POP_TOP_UNICODE_r32 1189 -#define _POP_TWO_r20 1190 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 1191 -#define _PUSH_EXC_INFO_r02 1192 -#define _PUSH_EXC_INFO_r12 1193 -#define _PUSH_EXC_INFO_r23 1194 -#define _PUSH_FRAME_r10 1195 -#define _PUSH_NULL_r01 1196 -#define _PUSH_NULL_r12 1197 -#define _PUSH_NULL_r23 1198 -#define _PUSH_NULL_CONDITIONAL_r00 1199 -#define _PY_FRAME_GENERAL_r01 1200 -#define _PY_FRAME_KW_r11 1201 -#define _QUICKEN_RESUME_r00 1202 -#define _QUICKEN_RESUME_r11 1203 -#define _QUICKEN_RESUME_r22 1204 -#define _QUICKEN_RESUME_r33 1205 -#define _REPLACE_WITH_TRUE_r11 1206 -#define _RESUME_CHECK_r00 1207 -#define _RESUME_CHECK_r11 1208 -#define _RESUME_CHECK_r22 1209 -#define _RESUME_CHECK_r33 1210 -#define _RETURN_GENERATOR_r01 1211 -#define _RETURN_VALUE_r11 1212 -#define _SAVE_RETURN_OFFSET_r00 1213 -#define _SAVE_RETURN_OFFSET_r11 1214 -#define _SAVE_RETURN_OFFSET_r22 1215 -#define _SAVE_RETURN_OFFSET_r33 1216 -#define _SEND_r22 1217 -#define _SEND_GEN_FRAME_r22 1218 -#define _SETUP_ANNOTATIONS_r00 1219 -#define _SET_ADD_r10 1220 -#define _SET_FUNCTION_ATTRIBUTE_r01 1221 -#define _SET_FUNCTION_ATTRIBUTE_r11 1222 -#define _SET_FUNCTION_ATTRIBUTE_r21 1223 -#define _SET_FUNCTION_ATTRIBUTE_r32 1224 -#define _SET_IP_r00 1225 -#define _SET_IP_r11 1226 -#define _SET_IP_r22 1227 -#define _SET_IP_r33 1228 -#define _SET_UPDATE_r10 1229 -#define _SPILL_OR_RELOAD_r01 1230 -#define _SPILL_OR_RELOAD_r02 1231 -#define _SPILL_OR_RELOAD_r03 1232 -#define _SPILL_OR_RELOAD_r10 1233 -#define _SPILL_OR_RELOAD_r12 1234 -#define _SPILL_OR_RELOAD_r13 1235 -#define _SPILL_OR_RELOAD_r20 1236 -#define _SPILL_OR_RELOAD_r21 1237 -#define _SPILL_OR_RELOAD_r23 1238 -#define _SPILL_OR_RELOAD_r30 1239 -#define _SPILL_OR_RELOAD_r31 1240 -#define _SPILL_OR_RELOAD_r32 1241 -#define _START_EXECUTOR_r00 1242 -#define _STORE_ATTR_r20 1243 -#define _STORE_ATTR_INSTANCE_VALUE_r20 1244 -#define _STORE_ATTR_SLOT_r20 1245 -#define _STORE_ATTR_WITH_HINT_r20 1246 -#define _STORE_DEREF_r10 1247 -#define _STORE_FAST_r10 1248 -#define _STORE_FAST_0_r10 1249 -#define _STORE_FAST_1_r10 1250 -#define _STORE_FAST_2_r10 1251 -#define _STORE_FAST_3_r10 1252 -#define _STORE_FAST_4_r10 1253 -#define _STORE_FAST_5_r10 1254 -#define _STORE_FAST_6_r10 1255 -#define _STORE_FAST_7_r10 1256 -#define _STORE_FAST_LOAD_FAST_r11 1257 -#define _STORE_FAST_STORE_FAST_r20 1258 -#define _STORE_GLOBAL_r10 1259 -#define _STORE_NAME_r10 1260 -#define _STORE_SLICE_r30 1261 -#define _STORE_SUBSCR_r30 1262 -#define _STORE_SUBSCR_DICT_r30 1263 -#define _STORE_SUBSCR_LIST_INT_r30 1264 -#define _SWAP_r11 1265 -#define _SWAP_2_r02 1266 -#define _SWAP_2_r12 1267 -#define _SWAP_2_r22 1268 -#define _SWAP_2_r33 1269 -#define _SWAP_3_r03 1270 -#define _SWAP_3_r13 1271 -#define _SWAP_3_r23 1272 -#define _SWAP_3_r33 1273 -#define _TIER2_RESUME_CHECK_r00 1274 -#define _TIER2_RESUME_CHECK_r11 1275 -#define _TIER2_RESUME_CHECK_r22 1276 -#define _TIER2_RESUME_CHECK_r33 1277 -#define _TO_BOOL_r11 1278 -#define _TO_BOOL_BOOL_r01 1279 -#define _TO_BOOL_BOOL_r11 1280 -#define _TO_BOOL_BOOL_r22 1281 -#define _TO_BOOL_BOOL_r33 1282 -#define _TO_BOOL_INT_r11 1283 -#define _TO_BOOL_LIST_r11 1284 -#define _TO_BOOL_NONE_r01 1285 -#define _TO_BOOL_NONE_r11 1286 -#define _TO_BOOL_NONE_r22 1287 -#define _TO_BOOL_NONE_r33 1288 -#define _TO_BOOL_STR_r11 1289 -#define _TRACE_RECORD_r00 1290 -#define _UNARY_INVERT_r11 1291 -#define _UNARY_NEGATIVE_r11 1292 -#define _UNARY_NOT_r01 1293 -#define _UNARY_NOT_r11 1294 -#define _UNARY_NOT_r22 1295 -#define _UNARY_NOT_r33 1296 -#define _UNPACK_EX_r10 1297 -#define _UNPACK_SEQUENCE_r10 1298 -#define _UNPACK_SEQUENCE_LIST_r10 1299 -#define _UNPACK_SEQUENCE_TUPLE_r10 1300 -#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1301 -#define _WITH_EXCEPT_START_r33 1302 -#define _YIELD_VALUE_r11 1303 -#define MAX_UOP_REGS_ID 1303 +#define _FOR_ITER_GEN_FRAME_r03 749 +#define _FOR_ITER_GEN_FRAME_r13 750 +#define _FOR_ITER_GEN_FRAME_r23 751 +#define _FOR_ITER_TIER_TWO_r23 752 +#define _GET_AITER_r11 753 +#define _GET_ANEXT_r12 754 +#define _GET_AWAITABLE_r11 755 +#define _GET_ITER_r12 756 +#define _GET_LEN_r12 757 +#define _GET_YIELD_FROM_ITER_r11 758 +#define _GUARD_BINARY_OP_EXTEND_r22 759 +#define _GUARD_CALLABLE_ISINSTANCE_r03 760 +#define _GUARD_CALLABLE_ISINSTANCE_r13 761 +#define _GUARD_CALLABLE_ISINSTANCE_r23 762 +#define _GUARD_CALLABLE_ISINSTANCE_r33 763 +#define _GUARD_CALLABLE_LEN_r03 764 +#define _GUARD_CALLABLE_LEN_r13 765 +#define _GUARD_CALLABLE_LEN_r23 766 +#define _GUARD_CALLABLE_LEN_r33 767 +#define _GUARD_CALLABLE_LIST_APPEND_r03 768 +#define _GUARD_CALLABLE_LIST_APPEND_r13 769 +#define _GUARD_CALLABLE_LIST_APPEND_r23 770 +#define _GUARD_CALLABLE_LIST_APPEND_r33 771 +#define _GUARD_CALLABLE_STR_1_r03 772 +#define _GUARD_CALLABLE_STR_1_r13 773 +#define _GUARD_CALLABLE_STR_1_r23 774 +#define _GUARD_CALLABLE_STR_1_r33 775 +#define _GUARD_CALLABLE_TUPLE_1_r03 776 +#define _GUARD_CALLABLE_TUPLE_1_r13 777 +#define _GUARD_CALLABLE_TUPLE_1_r23 778 +#define _GUARD_CALLABLE_TUPLE_1_r33 779 +#define _GUARD_CALLABLE_TYPE_1_r03 780 +#define _GUARD_CALLABLE_TYPE_1_r13 781 +#define _GUARD_CALLABLE_TYPE_1_r23 782 +#define _GUARD_CALLABLE_TYPE_1_r33 783 +#define _GUARD_DORV_NO_DICT_r01 784 +#define _GUARD_DORV_NO_DICT_r11 785 +#define _GUARD_DORV_NO_DICT_r22 786 +#define _GUARD_DORV_NO_DICT_r33 787 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01 788 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11 789 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22 790 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 791 +#define _GUARD_GLOBALS_VERSION_r00 792 +#define _GUARD_GLOBALS_VERSION_r11 793 +#define _GUARD_GLOBALS_VERSION_r22 794 +#define _GUARD_GLOBALS_VERSION_r33 795 +#define _GUARD_IP_RETURN_GENERATOR_r00 796 +#define _GUARD_IP_RETURN_GENERATOR_r11 797 +#define _GUARD_IP_RETURN_GENERATOR_r22 798 +#define _GUARD_IP_RETURN_GENERATOR_r33 799 +#define _GUARD_IP_RETURN_VALUE_r00 800 +#define _GUARD_IP_RETURN_VALUE_r11 801 +#define _GUARD_IP_RETURN_VALUE_r22 802 +#define _GUARD_IP_RETURN_VALUE_r33 803 +#define _GUARD_IP_YIELD_VALUE_r00 804 +#define _GUARD_IP_YIELD_VALUE_r11 805 +#define _GUARD_IP_YIELD_VALUE_r22 806 +#define _GUARD_IP_YIELD_VALUE_r33 807 +#define _GUARD_IP__PUSH_FRAME_r00 808 +#define _GUARD_IP__PUSH_FRAME_r11 809 +#define _GUARD_IP__PUSH_FRAME_r22 810 +#define _GUARD_IP__PUSH_FRAME_r33 811 +#define _GUARD_IS_FALSE_POP_r00 812 +#define _GUARD_IS_FALSE_POP_r10 813 +#define _GUARD_IS_FALSE_POP_r21 814 +#define _GUARD_IS_FALSE_POP_r32 815 +#define _GUARD_IS_NONE_POP_r00 816 +#define _GUARD_IS_NONE_POP_r10 817 +#define _GUARD_IS_NONE_POP_r21 818 +#define _GUARD_IS_NONE_POP_r32 819 +#define _GUARD_IS_NOT_NONE_POP_r10 820 +#define _GUARD_IS_TRUE_POP_r00 821 +#define _GUARD_IS_TRUE_POP_r10 822 +#define _GUARD_IS_TRUE_POP_r21 823 +#define _GUARD_IS_TRUE_POP_r32 824 +#define _GUARD_KEYS_VERSION_r01 825 +#define _GUARD_KEYS_VERSION_r11 826 +#define _GUARD_KEYS_VERSION_r22 827 +#define _GUARD_KEYS_VERSION_r33 828 +#define _GUARD_NOS_DICT_r02 829 +#define _GUARD_NOS_DICT_r12 830 +#define _GUARD_NOS_DICT_r22 831 +#define _GUARD_NOS_DICT_r33 832 +#define _GUARD_NOS_FLOAT_r02 833 +#define _GUARD_NOS_FLOAT_r12 834 +#define _GUARD_NOS_FLOAT_r22 835 +#define _GUARD_NOS_FLOAT_r33 836 +#define _GUARD_NOS_INT_r02 837 +#define _GUARD_NOS_INT_r12 838 +#define _GUARD_NOS_INT_r22 839 +#define _GUARD_NOS_INT_r33 840 +#define _GUARD_NOS_LIST_r02 841 +#define _GUARD_NOS_LIST_r12 842 +#define _GUARD_NOS_LIST_r22 843 +#define _GUARD_NOS_LIST_r33 844 +#define _GUARD_NOS_NOT_NULL_r02 845 +#define _GUARD_NOS_NOT_NULL_r12 846 +#define _GUARD_NOS_NOT_NULL_r22 847 +#define _GUARD_NOS_NOT_NULL_r33 848 +#define _GUARD_NOS_NULL_r02 849 +#define _GUARD_NOS_NULL_r12 850 +#define _GUARD_NOS_NULL_r22 851 +#define _GUARD_NOS_NULL_r33 852 +#define _GUARD_NOS_OVERFLOWED_r02 853 +#define _GUARD_NOS_OVERFLOWED_r12 854 +#define _GUARD_NOS_OVERFLOWED_r22 855 +#define _GUARD_NOS_OVERFLOWED_r33 856 +#define _GUARD_NOS_TUPLE_r02 857 +#define _GUARD_NOS_TUPLE_r12 858 +#define _GUARD_NOS_TUPLE_r22 859 +#define _GUARD_NOS_TUPLE_r33 860 +#define _GUARD_NOS_UNICODE_r02 861 +#define _GUARD_NOS_UNICODE_r12 862 +#define _GUARD_NOS_UNICODE_r22 863 +#define _GUARD_NOS_UNICODE_r33 864 +#define _GUARD_NOT_EXHAUSTED_LIST_r02 865 +#define _GUARD_NOT_EXHAUSTED_LIST_r12 866 +#define _GUARD_NOT_EXHAUSTED_LIST_r22 867 +#define _GUARD_NOT_EXHAUSTED_LIST_r33 868 +#define _GUARD_NOT_EXHAUSTED_RANGE_r02 869 +#define _GUARD_NOT_EXHAUSTED_RANGE_r12 870 +#define _GUARD_NOT_EXHAUSTED_RANGE_r22 871 +#define _GUARD_NOT_EXHAUSTED_RANGE_r33 872 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r02 873 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r12 874 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r22 875 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r33 876 +#define _GUARD_THIRD_NULL_r03 877 +#define _GUARD_THIRD_NULL_r13 878 +#define _GUARD_THIRD_NULL_r23 879 +#define _GUARD_THIRD_NULL_r33 880 +#define _GUARD_TOS_ANY_SET_r01 881 +#define _GUARD_TOS_ANY_SET_r11 882 +#define _GUARD_TOS_ANY_SET_r22 883 +#define _GUARD_TOS_ANY_SET_r33 884 +#define _GUARD_TOS_DICT_r01 885 +#define _GUARD_TOS_DICT_r11 886 +#define _GUARD_TOS_DICT_r22 887 +#define _GUARD_TOS_DICT_r33 888 +#define _GUARD_TOS_FLOAT_r01 889 +#define _GUARD_TOS_FLOAT_r11 890 +#define _GUARD_TOS_FLOAT_r22 891 +#define _GUARD_TOS_FLOAT_r33 892 +#define _GUARD_TOS_INT_r01 893 +#define _GUARD_TOS_INT_r11 894 +#define _GUARD_TOS_INT_r22 895 +#define _GUARD_TOS_INT_r33 896 +#define _GUARD_TOS_LIST_r01 897 +#define _GUARD_TOS_LIST_r11 898 +#define _GUARD_TOS_LIST_r22 899 +#define _GUARD_TOS_LIST_r33 900 +#define _GUARD_TOS_OVERFLOWED_r01 901 +#define _GUARD_TOS_OVERFLOWED_r11 902 +#define _GUARD_TOS_OVERFLOWED_r22 903 +#define _GUARD_TOS_OVERFLOWED_r33 904 +#define _GUARD_TOS_SLICE_r01 905 +#define _GUARD_TOS_SLICE_r11 906 +#define _GUARD_TOS_SLICE_r22 907 +#define _GUARD_TOS_SLICE_r33 908 +#define _GUARD_TOS_TUPLE_r01 909 +#define _GUARD_TOS_TUPLE_r11 910 +#define _GUARD_TOS_TUPLE_r22 911 +#define _GUARD_TOS_TUPLE_r33 912 +#define _GUARD_TOS_UNICODE_r01 913 +#define _GUARD_TOS_UNICODE_r11 914 +#define _GUARD_TOS_UNICODE_r22 915 +#define _GUARD_TOS_UNICODE_r33 916 +#define _GUARD_TYPE_VERSION_r01 917 +#define _GUARD_TYPE_VERSION_r11 918 +#define _GUARD_TYPE_VERSION_r22 919 +#define _GUARD_TYPE_VERSION_r33 920 +#define _GUARD_TYPE_VERSION_AND_LOCK_r01 921 +#define _GUARD_TYPE_VERSION_AND_LOCK_r11 922 +#define _GUARD_TYPE_VERSION_AND_LOCK_r22 923 +#define _GUARD_TYPE_VERSION_AND_LOCK_r33 924 +#define _HANDLE_PENDING_AND_DEOPT_r00 925 +#define _HANDLE_PENDING_AND_DEOPT_r10 926 +#define _HANDLE_PENDING_AND_DEOPT_r20 927 +#define _HANDLE_PENDING_AND_DEOPT_r30 928 +#define _IMPORT_FROM_r12 929 +#define _IMPORT_NAME_r21 930 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 931 +#define _INIT_CALL_PY_EXACT_ARGS_r01 932 +#define _INIT_CALL_PY_EXACT_ARGS_0_r01 933 +#define _INIT_CALL_PY_EXACT_ARGS_1_r01 934 +#define _INIT_CALL_PY_EXACT_ARGS_2_r01 935 +#define _INIT_CALL_PY_EXACT_ARGS_3_r01 936 +#define _INIT_CALL_PY_EXACT_ARGS_4_r01 937 +#define _INSERT_NULL_r10 938 +#define _INSTRUMENTED_FOR_ITER_r23 939 +#define _INSTRUMENTED_INSTRUCTION_r00 940 +#define _INSTRUMENTED_JUMP_FORWARD_r00 941 +#define _INSTRUMENTED_JUMP_FORWARD_r11 942 +#define _INSTRUMENTED_JUMP_FORWARD_r22 943 +#define _INSTRUMENTED_JUMP_FORWARD_r33 944 +#define _INSTRUMENTED_LINE_r00 945 +#define _INSTRUMENTED_NOT_TAKEN_r00 946 +#define _INSTRUMENTED_NOT_TAKEN_r11 947 +#define _INSTRUMENTED_NOT_TAKEN_r22 948 +#define _INSTRUMENTED_NOT_TAKEN_r33 949 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r00 950 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 951 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 952 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 953 +#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 954 +#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 955 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r00 956 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 957 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 958 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 959 +#define _IS_NONE_r11 960 +#define _IS_OP_r21 961 +#define _ITER_CHECK_LIST_r02 962 +#define _ITER_CHECK_LIST_r12 963 +#define _ITER_CHECK_LIST_r22 964 +#define _ITER_CHECK_LIST_r33 965 +#define _ITER_CHECK_RANGE_r02 966 +#define _ITER_CHECK_RANGE_r12 967 +#define _ITER_CHECK_RANGE_r22 968 +#define _ITER_CHECK_RANGE_r33 969 +#define _ITER_CHECK_TUPLE_r02 970 +#define _ITER_CHECK_TUPLE_r12 971 +#define _ITER_CHECK_TUPLE_r22 972 +#define _ITER_CHECK_TUPLE_r33 973 +#define _ITER_JUMP_LIST_r02 974 +#define _ITER_JUMP_LIST_r12 975 +#define _ITER_JUMP_LIST_r22 976 +#define _ITER_JUMP_LIST_r33 977 +#define _ITER_JUMP_RANGE_r02 978 +#define _ITER_JUMP_RANGE_r12 979 +#define _ITER_JUMP_RANGE_r22 980 +#define _ITER_JUMP_RANGE_r33 981 +#define _ITER_JUMP_TUPLE_r02 982 +#define _ITER_JUMP_TUPLE_r12 983 +#define _ITER_JUMP_TUPLE_r22 984 +#define _ITER_JUMP_TUPLE_r33 985 +#define _ITER_NEXT_LIST_r23 986 +#define _ITER_NEXT_LIST_TIER_TWO_r23 987 +#define _ITER_NEXT_RANGE_r03 988 +#define _ITER_NEXT_RANGE_r13 989 +#define _ITER_NEXT_RANGE_r23 990 +#define _ITER_NEXT_TUPLE_r03 991 +#define _ITER_NEXT_TUPLE_r13 992 +#define _ITER_NEXT_TUPLE_r23 993 +#define _JUMP_BACKWARD_NO_INTERRUPT_r00 994 +#define _JUMP_BACKWARD_NO_INTERRUPT_r11 995 +#define _JUMP_BACKWARD_NO_INTERRUPT_r22 996 +#define _JUMP_BACKWARD_NO_INTERRUPT_r33 997 +#define _JUMP_TO_TOP_r00 998 +#define _LIST_APPEND_r10 999 +#define _LIST_EXTEND_r10 1000 +#define _LOAD_ATTR_r10 1001 +#define _LOAD_ATTR_CLASS_r11 1002 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_r11 1003 +#define _LOAD_ATTR_INSTANCE_VALUE_r11 1004 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r02 1005 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 1006 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 1007 +#define _LOAD_ATTR_METHOD_NO_DICT_r02 1008 +#define _LOAD_ATTR_METHOD_NO_DICT_r12 1009 +#define _LOAD_ATTR_METHOD_NO_DICT_r23 1010 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r02 1011 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 1012 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 1013 +#define _LOAD_ATTR_MODULE_r11 1014 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1015 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1016 +#define _LOAD_ATTR_PROPERTY_FRAME_r11 1017 +#define _LOAD_ATTR_SLOT_r11 1018 +#define _LOAD_ATTR_WITH_HINT_r11 1019 +#define _LOAD_BUILD_CLASS_r01 1020 +#define _LOAD_BYTECODE_r00 1021 +#define _LOAD_COMMON_CONSTANT_r01 1022 +#define _LOAD_COMMON_CONSTANT_r12 1023 +#define _LOAD_COMMON_CONSTANT_r23 1024 +#define _LOAD_CONST_r01 1025 +#define _LOAD_CONST_r12 1026 +#define _LOAD_CONST_r23 1027 +#define _LOAD_CONST_INLINE_r01 1028 +#define _LOAD_CONST_INLINE_r12 1029 +#define _LOAD_CONST_INLINE_r23 1030 +#define _LOAD_CONST_INLINE_BORROW_r01 1031 +#define _LOAD_CONST_INLINE_BORROW_r12 1032 +#define _LOAD_CONST_INLINE_BORROW_r23 1033 +#define _LOAD_CONST_UNDER_INLINE_r02 1034 +#define _LOAD_CONST_UNDER_INLINE_r12 1035 +#define _LOAD_CONST_UNDER_INLINE_r23 1036 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r02 1037 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r12 1038 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r23 1039 +#define _LOAD_DEREF_r01 1040 +#define _LOAD_FAST_r01 1041 +#define _LOAD_FAST_r12 1042 +#define _LOAD_FAST_r23 1043 +#define _LOAD_FAST_0_r01 1044 +#define _LOAD_FAST_0_r12 1045 +#define _LOAD_FAST_0_r23 1046 +#define _LOAD_FAST_1_r01 1047 +#define _LOAD_FAST_1_r12 1048 +#define _LOAD_FAST_1_r23 1049 +#define _LOAD_FAST_2_r01 1050 +#define _LOAD_FAST_2_r12 1051 +#define _LOAD_FAST_2_r23 1052 +#define _LOAD_FAST_3_r01 1053 +#define _LOAD_FAST_3_r12 1054 +#define _LOAD_FAST_3_r23 1055 +#define _LOAD_FAST_4_r01 1056 +#define _LOAD_FAST_4_r12 1057 +#define _LOAD_FAST_4_r23 1058 +#define _LOAD_FAST_5_r01 1059 +#define _LOAD_FAST_5_r12 1060 +#define _LOAD_FAST_5_r23 1061 +#define _LOAD_FAST_6_r01 1062 +#define _LOAD_FAST_6_r12 1063 +#define _LOAD_FAST_6_r23 1064 +#define _LOAD_FAST_7_r01 1065 +#define _LOAD_FAST_7_r12 1066 +#define _LOAD_FAST_7_r23 1067 +#define _LOAD_FAST_AND_CLEAR_r01 1068 +#define _LOAD_FAST_AND_CLEAR_r12 1069 +#define _LOAD_FAST_AND_CLEAR_r23 1070 +#define _LOAD_FAST_BORROW_r01 1071 +#define _LOAD_FAST_BORROW_r12 1072 +#define _LOAD_FAST_BORROW_r23 1073 +#define _LOAD_FAST_BORROW_0_r01 1074 +#define _LOAD_FAST_BORROW_0_r12 1075 +#define _LOAD_FAST_BORROW_0_r23 1076 +#define _LOAD_FAST_BORROW_1_r01 1077 +#define _LOAD_FAST_BORROW_1_r12 1078 +#define _LOAD_FAST_BORROW_1_r23 1079 +#define _LOAD_FAST_BORROW_2_r01 1080 +#define _LOAD_FAST_BORROW_2_r12 1081 +#define _LOAD_FAST_BORROW_2_r23 1082 +#define _LOAD_FAST_BORROW_3_r01 1083 +#define _LOAD_FAST_BORROW_3_r12 1084 +#define _LOAD_FAST_BORROW_3_r23 1085 +#define _LOAD_FAST_BORROW_4_r01 1086 +#define _LOAD_FAST_BORROW_4_r12 1087 +#define _LOAD_FAST_BORROW_4_r23 1088 +#define _LOAD_FAST_BORROW_5_r01 1089 +#define _LOAD_FAST_BORROW_5_r12 1090 +#define _LOAD_FAST_BORROW_5_r23 1091 +#define _LOAD_FAST_BORROW_6_r01 1092 +#define _LOAD_FAST_BORROW_6_r12 1093 +#define _LOAD_FAST_BORROW_6_r23 1094 +#define _LOAD_FAST_BORROW_7_r01 1095 +#define _LOAD_FAST_BORROW_7_r12 1096 +#define _LOAD_FAST_BORROW_7_r23 1097 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1098 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1099 +#define _LOAD_FAST_CHECK_r01 1100 +#define _LOAD_FAST_LOAD_FAST_r02 1101 +#define _LOAD_FAST_LOAD_FAST_r13 1102 +#define _LOAD_FROM_DICT_OR_DEREF_r11 1103 +#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1104 +#define _LOAD_GLOBAL_r00 1105 +#define _LOAD_GLOBAL_BUILTINS_r01 1106 +#define _LOAD_GLOBAL_MODULE_r01 1107 +#define _LOAD_LOCALS_r01 1108 +#define _LOAD_NAME_r01 1109 +#define _LOAD_SMALL_INT_r01 1110 +#define _LOAD_SMALL_INT_r12 1111 +#define _LOAD_SMALL_INT_r23 1112 +#define _LOAD_SMALL_INT_0_r01 1113 +#define _LOAD_SMALL_INT_0_r12 1114 +#define _LOAD_SMALL_INT_0_r23 1115 +#define _LOAD_SMALL_INT_1_r01 1116 +#define _LOAD_SMALL_INT_1_r12 1117 +#define _LOAD_SMALL_INT_1_r23 1118 +#define _LOAD_SMALL_INT_2_r01 1119 +#define _LOAD_SMALL_INT_2_r12 1120 +#define _LOAD_SMALL_INT_2_r23 1121 +#define _LOAD_SMALL_INT_3_r01 1122 +#define _LOAD_SMALL_INT_3_r12 1123 +#define _LOAD_SMALL_INT_3_r23 1124 +#define _LOAD_SPECIAL_r00 1125 +#define _LOAD_SUPER_ATTR_ATTR_r31 1126 +#define _LOAD_SUPER_ATTR_METHOD_r32 1127 +#define _MAKE_CALLARGS_A_TUPLE_r33 1128 +#define _MAKE_CELL_r00 1129 +#define _MAKE_FUNCTION_r11 1130 +#define _MAKE_WARM_r00 1131 +#define _MAKE_WARM_r11 1132 +#define _MAKE_WARM_r22 1133 +#define _MAKE_WARM_r33 1134 +#define _MAP_ADD_r20 1135 +#define _MATCH_CLASS_r31 1136 +#define _MATCH_KEYS_r23 1137 +#define _MATCH_MAPPING_r02 1138 +#define _MATCH_MAPPING_r12 1139 +#define _MATCH_MAPPING_r23 1140 +#define _MATCH_SEQUENCE_r02 1141 +#define _MATCH_SEQUENCE_r12 1142 +#define _MATCH_SEQUENCE_r23 1143 +#define _MAYBE_EXPAND_METHOD_r00 1144 +#define _MAYBE_EXPAND_METHOD_KW_r11 1145 +#define _MONITOR_CALL_r00 1146 +#define _MONITOR_CALL_KW_r11 1147 +#define _MONITOR_JUMP_BACKWARD_r00 1148 +#define _MONITOR_JUMP_BACKWARD_r11 1149 +#define _MONITOR_JUMP_BACKWARD_r22 1150 +#define _MONITOR_JUMP_BACKWARD_r33 1151 +#define _MONITOR_RESUME_r00 1152 +#define _NOP_r00 1153 +#define _NOP_r11 1154 +#define _NOP_r22 1155 +#define _NOP_r33 1156 +#define _POP_CALL_r20 1157 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 1158 +#define _POP_CALL_ONE_r30 1159 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 1160 +#define _POP_CALL_TWO_r30 1161 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 1162 +#define _POP_EXCEPT_r10 1163 +#define _POP_ITER_r20 1164 +#define _POP_JUMP_IF_FALSE_r00 1165 +#define _POP_JUMP_IF_FALSE_r10 1166 +#define _POP_JUMP_IF_FALSE_r21 1167 +#define _POP_JUMP_IF_FALSE_r32 1168 +#define _POP_JUMP_IF_TRUE_r00 1169 +#define _POP_JUMP_IF_TRUE_r10 1170 +#define _POP_JUMP_IF_TRUE_r21 1171 +#define _POP_JUMP_IF_TRUE_r32 1172 +#define _POP_TOP_r10 1173 +#define _POP_TOP_FLOAT_r00 1174 +#define _POP_TOP_FLOAT_r10 1175 +#define _POP_TOP_FLOAT_r21 1176 +#define _POP_TOP_FLOAT_r32 1177 +#define _POP_TOP_INT_r00 1178 +#define _POP_TOP_INT_r10 1179 +#define _POP_TOP_INT_r21 1180 +#define _POP_TOP_INT_r32 1181 +#define _POP_TOP_LOAD_CONST_INLINE_r11 1182 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 1183 +#define _POP_TOP_NOP_r00 1184 +#define _POP_TOP_NOP_r10 1185 +#define _POP_TOP_NOP_r21 1186 +#define _POP_TOP_NOP_r32 1187 +#define _POP_TOP_UNICODE_r00 1188 +#define _POP_TOP_UNICODE_r10 1189 +#define _POP_TOP_UNICODE_r21 1190 +#define _POP_TOP_UNICODE_r32 1191 +#define _POP_TWO_r20 1192 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 1193 +#define _PUSH_EXC_INFO_r02 1194 +#define _PUSH_EXC_INFO_r12 1195 +#define _PUSH_EXC_INFO_r23 1196 +#define _PUSH_FRAME_r10 1197 +#define _PUSH_NULL_r01 1198 +#define _PUSH_NULL_r12 1199 +#define _PUSH_NULL_r23 1200 +#define _PUSH_NULL_CONDITIONAL_r00 1201 +#define _PY_FRAME_GENERAL_r01 1202 +#define _PY_FRAME_KW_r11 1203 +#define _QUICKEN_RESUME_r00 1204 +#define _QUICKEN_RESUME_r11 1205 +#define _QUICKEN_RESUME_r22 1206 +#define _QUICKEN_RESUME_r33 1207 +#define _REPLACE_WITH_TRUE_r11 1208 +#define _RESUME_CHECK_r00 1209 +#define _RESUME_CHECK_r11 1210 +#define _RESUME_CHECK_r22 1211 +#define _RESUME_CHECK_r33 1212 +#define _RETURN_GENERATOR_r01 1213 +#define _RETURN_VALUE_r11 1214 +#define _SAVE_RETURN_OFFSET_r00 1215 +#define _SAVE_RETURN_OFFSET_r11 1216 +#define _SAVE_RETURN_OFFSET_r22 1217 +#define _SAVE_RETURN_OFFSET_r33 1218 +#define _SEND_r22 1219 +#define _SEND_GEN_FRAME_r22 1220 +#define _SETUP_ANNOTATIONS_r00 1221 +#define _SET_ADD_r10 1222 +#define _SET_FUNCTION_ATTRIBUTE_r01 1223 +#define _SET_FUNCTION_ATTRIBUTE_r11 1224 +#define _SET_FUNCTION_ATTRIBUTE_r21 1225 +#define _SET_FUNCTION_ATTRIBUTE_r32 1226 +#define _SET_IP_r00 1227 +#define _SET_IP_r11 1228 +#define _SET_IP_r22 1229 +#define _SET_IP_r33 1230 +#define _SET_UPDATE_r10 1231 +#define _SPILL_OR_RELOAD_r01 1232 +#define _SPILL_OR_RELOAD_r02 1233 +#define _SPILL_OR_RELOAD_r03 1234 +#define _SPILL_OR_RELOAD_r10 1235 +#define _SPILL_OR_RELOAD_r12 1236 +#define _SPILL_OR_RELOAD_r13 1237 +#define _SPILL_OR_RELOAD_r20 1238 +#define _SPILL_OR_RELOAD_r21 1239 +#define _SPILL_OR_RELOAD_r23 1240 +#define _SPILL_OR_RELOAD_r30 1241 +#define _SPILL_OR_RELOAD_r31 1242 +#define _SPILL_OR_RELOAD_r32 1243 +#define _START_EXECUTOR_r00 1244 +#define _STORE_ATTR_r20 1245 +#define _STORE_ATTR_INSTANCE_VALUE_r20 1246 +#define _STORE_ATTR_SLOT_r20 1247 +#define _STORE_ATTR_WITH_HINT_r20 1248 +#define _STORE_DEREF_r10 1249 +#define _STORE_FAST_r10 1250 +#define _STORE_FAST_0_r10 1251 +#define _STORE_FAST_1_r10 1252 +#define _STORE_FAST_2_r10 1253 +#define _STORE_FAST_3_r10 1254 +#define _STORE_FAST_4_r10 1255 +#define _STORE_FAST_5_r10 1256 +#define _STORE_FAST_6_r10 1257 +#define _STORE_FAST_7_r10 1258 +#define _STORE_FAST_LOAD_FAST_r11 1259 +#define _STORE_FAST_STORE_FAST_r20 1260 +#define _STORE_GLOBAL_r10 1261 +#define _STORE_NAME_r10 1262 +#define _STORE_SLICE_r30 1263 +#define _STORE_SUBSCR_r30 1264 +#define _STORE_SUBSCR_DICT_r30 1265 +#define _STORE_SUBSCR_LIST_INT_r30 1266 +#define _SWAP_r11 1267 +#define _SWAP_2_r02 1268 +#define _SWAP_2_r12 1269 +#define _SWAP_2_r22 1270 +#define _SWAP_2_r33 1271 +#define _SWAP_3_r03 1272 +#define _SWAP_3_r13 1273 +#define _SWAP_3_r23 1274 +#define _SWAP_3_r33 1275 +#define _TIER2_RESUME_CHECK_r00 1276 +#define _TIER2_RESUME_CHECK_r11 1277 +#define _TIER2_RESUME_CHECK_r22 1278 +#define _TIER2_RESUME_CHECK_r33 1279 +#define _TO_BOOL_r11 1280 +#define _TO_BOOL_BOOL_r01 1281 +#define _TO_BOOL_BOOL_r11 1282 +#define _TO_BOOL_BOOL_r22 1283 +#define _TO_BOOL_BOOL_r33 1284 +#define _TO_BOOL_INT_r11 1285 +#define _TO_BOOL_LIST_r11 1286 +#define _TO_BOOL_NONE_r01 1287 +#define _TO_BOOL_NONE_r11 1288 +#define _TO_BOOL_NONE_r22 1289 +#define _TO_BOOL_NONE_r33 1290 +#define _TO_BOOL_STR_r11 1291 +#define _TRACE_RECORD_r00 1292 +#define _UNARY_INVERT_r11 1293 +#define _UNARY_NEGATIVE_r11 1294 +#define _UNARY_NOT_r01 1295 +#define _UNARY_NOT_r11 1296 +#define _UNARY_NOT_r22 1297 +#define _UNARY_NOT_r33 1298 +#define _UNPACK_EX_r10 1299 +#define _UNPACK_SEQUENCE_r10 1300 +#define _UNPACK_SEQUENCE_LIST_r10 1301 +#define _UNPACK_SEQUENCE_TUPLE_r10 1302 +#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1303 +#define _WITH_EXCEPT_START_r33 1304 +#define _YIELD_VALUE_r11 1305 +#define MAX_UOP_REGS_ID 1305 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 1e74588d3aa62e..14627cc2ca7523 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -2160,10 +2160,10 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { }, }, [_FOR_ITER_GEN_FRAME] = { - .best = { 2, 2, 2, 2 }, + .best = { 0, 1, 2, 2 }, .entries = { - { -1, -1, -1 }, - { -1, -1, -1 }, + { 3, 0, _FOR_ITER_GEN_FRAME_r03 }, + { 3, 1, _FOR_ITER_GEN_FRAME_r13 }, { 3, 2, _FOR_ITER_GEN_FRAME_r23 }, { -1, -1, -1 }, }, @@ -3653,6 +3653,8 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_ITER_NEXT_RANGE_r03] = _ITER_NEXT_RANGE, [_ITER_NEXT_RANGE_r13] = _ITER_NEXT_RANGE, [_ITER_NEXT_RANGE_r23] = _ITER_NEXT_RANGE, + [_FOR_ITER_GEN_FRAME_r03] = _FOR_ITER_GEN_FRAME, + [_FOR_ITER_GEN_FRAME_r13] = _FOR_ITER_GEN_FRAME, [_FOR_ITER_GEN_FRAME_r23] = _FOR_ITER_GEN_FRAME, [_INSERT_NULL_r10] = _INSERT_NULL, [_LOAD_SPECIAL_r00] = _LOAD_SPECIAL, @@ -4227,6 +4229,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_FORMAT_WITH_SPEC] = "_FORMAT_WITH_SPEC", [_FORMAT_WITH_SPEC_r21] = "_FORMAT_WITH_SPEC_r21", [_FOR_ITER_GEN_FRAME] = "_FOR_ITER_GEN_FRAME", + [_FOR_ITER_GEN_FRAME_r03] = "_FOR_ITER_GEN_FRAME_r03", + [_FOR_ITER_GEN_FRAME_r13] = "_FOR_ITER_GEN_FRAME_r13", [_FOR_ITER_GEN_FRAME_r23] = "_FOR_ITER_GEN_FRAME_r23", [_FOR_ITER_TIER_TWO] = "_FOR_ITER_TIER_TWO", [_FOR_ITER_TIER_TWO_r23] = "_FOR_ITER_TIER_TWO_r23", diff --git a/Modules/_remote_debugging/_remote_debugging.h b/Modules/_remote_debugging/_remote_debugging.h index 0aa98349296b8a..71914b00f4c215 100644 --- a/Modules/_remote_debugging/_remote_debugging.h +++ b/Modules/_remote_debugging/_remote_debugging.h @@ -20,7 +20,7 @@ extern "C" { #include "Python.h" #include // _Py_DebugOffsets -#include // FRAME_SUSPENDED_YIELD_FROM +#include // FRAME_SUSPENDED_YIELD_FROM #include // FRAME_OWNED_BY_INTERPRETER #include // struct llist_node #include // _PyLong_GetZero diff --git a/Objects/genobject.c b/Objects/genobject.c index e47180a23d2626..de4206e9a1330b 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -36,6 +36,14 @@ static PyObject* async_gen_athrow_new(PyAsyncGenObject *, PyObject *); #define _PyAsyncGenObject_CAST(op) \ _Py_CAST(PyAsyncGenObject*, (op)) +#ifdef Py_GIL_DISABLED +# define _Py_GEN_SET_FRAME_STATE(gen, expected, state) \ + _Py_atomic_compare_exchange_int8(&(gen)->gi_frame_state, &expected, (state)) +#else +# define _Py_GEN_SET_FRAME_STATE(gen, expected, state) \ + ((gen)->gi_frame_state = (state), true) +#endif + static const char *NON_INIT_CORO_MSG = "can't send non-None value to a " "just-started coroutine"; @@ -145,10 +153,6 @@ _PyGen_Finalize(PyObject *self) static void gen_clear_frame(PyGenObject *gen) { - if (gen->gi_frame_state == FRAME_CLEARED) - return; - - gen->gi_frame_state = FRAME_CLEARED; _PyInterpreterFrame *frame = &gen->gi_iframe; frame->previous = NULL; _PyFrame_ClearExceptCode(frame); @@ -179,7 +183,10 @@ gen_dealloc(PyObject *self) if (PyCoro_CheckExact(gen)) { Py_CLEAR(((PyCoroObject *)gen)->cr_origin_or_finalizer); } - gen_clear_frame(gen); + if (gen->gi_frame_state != FRAME_CLEARED) { + gen->gi_frame_state = FRAME_CLEARED; + gen_clear_frame(gen); + } assert(gen->gi_exc_state.exc_value == NULL); PyStackRef_CLEAR(gen->gi_iframe.f_executable); Py_CLEAR(gen->gi_name); @@ -188,58 +195,31 @@ gen_dealloc(PyObject *self) PyObject_GC_Del(gen); } -static PySendResult -gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, - int exc, int closing) +static void +gen_raise_already_executing_error(PyGenObject *gen) { - PyThreadState *tstate = _PyThreadState_GET(); - _PyInterpreterFrame *frame = &gen->gi_iframe; - - *presult = NULL; - if (gen->gi_frame_state == FRAME_CREATED && arg && arg != Py_None) { - const char *msg = "can't send non-None value to a " - "just-started generator"; - if (PyCoro_CheckExact(gen)) { - msg = NON_INIT_CORO_MSG; - } - else if (PyAsyncGen_CheckExact(gen)) { - msg = "can't send non-None value to a " - "just-started async generator"; - } - PyErr_SetString(PyExc_TypeError, msg); - return PYGEN_ERROR; + const char *msg = "generator already executing"; + if (PyCoro_CheckExact(gen)) { + msg = "coroutine already executing"; } - if (gen->gi_frame_state == FRAME_EXECUTING) { - const char *msg = "generator already executing"; - if (PyCoro_CheckExact(gen)) { - msg = "coroutine already executing"; - } - else if (PyAsyncGen_CheckExact(gen)) { - msg = "async generator already executing"; - } - PyErr_SetString(PyExc_ValueError, msg); - return PYGEN_ERROR; - } - if (FRAME_STATE_FINISHED(gen->gi_frame_state)) { - if (PyCoro_CheckExact(gen) && !closing) { - /* `gen` is an exhausted coroutine: raise an error, - except when called from gen_close(), which should - always be a silent method. */ - PyErr_SetString( - PyExc_RuntimeError, - "cannot reuse already awaited coroutine"); - } - else if (arg && !exc) { - /* `gen` is an exhausted generator: - only return value if called from send(). */ - *presult = Py_NewRef(Py_None); - return PYGEN_RETURN; - } - return PYGEN_ERROR; + else if (PyAsyncGen_CheckExact(gen)) { + msg = "async generator already executing"; } + PyErr_SetString(PyExc_ValueError, msg); +} + +// Send 'arg' into 'gen'. On success, return PYGEN_NEXT or PYGEN_RETURN. +// Returns PYGEN_ERROR on failure. 'presult' is set to the yielded or +// returned value. +// The generator must be in the FRAME_EXECUTING state when this function +// is called. +static PySendResult +gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, int exc) +{ + assert(FT_ATOMIC_LOAD_INT8_RELAXED(gen->gi_frame_state) == FRAME_EXECUTING); - assert((gen->gi_frame_state == FRAME_CREATED) || - FRAME_STATE_SUSPENDED(gen->gi_frame_state)); + PyThreadState *tstate = _PyThreadState_GET(); + _PyInterpreterFrame *frame = &gen->gi_iframe; /* Push arg onto the frame's value stack */ PyObject *arg_obj = arg ? arg : Py_None; @@ -254,18 +234,29 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, _PyErr_ChainStackItem(); } - gen->gi_frame_state = FRAME_EXECUTING; +#if defined(Py_GIL_DISABLED) && defined(Py_DEBUG) + ((_PyThreadStateImpl *)tstate)->gen_last_frame_state = FRAME_EXECUTING; +#endif + EVAL_CALL_STAT_INC(EVAL_CALL_GENERATOR); PyObject *result = _PyEval_EvalFrame(tstate, frame, exc); assert(tstate->exc_info == prev_exc_info); +#ifdef Py_GIL_DISABLED + // Grab the last frame state from the thread state instead of the + // generator, as it may have changed if another thread resumed this + // generator. + int8_t frame_state = ((_PyThreadStateImpl *)tstate)->gen_last_frame_state; +#else assert(gen->gi_exc_state.previous_item == NULL); - assert(gen->gi_frame_state != FRAME_EXECUTING); assert(frame->previous == NULL); + int8_t frame_state = gen->gi_frame_state; +#endif + assert(frame_state != FRAME_EXECUTING); /* If the generator just returned (as opposed to yielding), signal * that the generator is exhausted. */ if (result) { - if (FRAME_STATE_SUSPENDED(gen->gi_frame_state)) { + if (FRAME_STATE_SUSPENDED(frame_state)) { *presult = result; return PYGEN_NEXT; } @@ -282,36 +273,86 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, } assert(gen->gi_exc_state.exc_value == NULL); - assert(gen->gi_frame_state == FRAME_CLEARED); + assert(frame_state == FRAME_CLEARED); *presult = result; return result ? PYGEN_RETURN : PYGEN_ERROR; } +// Set the generator 'gen' to the executing state and send 'arg' into it. +// See gen_send_ex2() for details. +static PySendResult +gen_send_ex(PyGenObject *gen, PyObject *arg, PyObject **presult) +{ + *presult = NULL; + int8_t frame_state = FT_ATOMIC_LOAD_INT8_RELAXED(gen->gi_frame_state); +retry: + if (frame_state == FRAME_CREATED && arg && arg != Py_None) { + const char *msg = "can't send non-None value to a " + "just-started generator"; + if (PyCoro_CheckExact(gen)) { + msg = NON_INIT_CORO_MSG; + } + else if (PyAsyncGen_CheckExact(gen)) { + msg = "can't send non-None value to a " + "just-started async generator"; + } + PyErr_SetString(PyExc_TypeError, msg); + return PYGEN_ERROR; + } + if (frame_state == FRAME_EXECUTING) { + gen_raise_already_executing_error(gen); + return PYGEN_ERROR; + } + if (FRAME_STATE_FINISHED(frame_state)) { + if (PyCoro_CheckExact(gen)) { + /* `gen` is an exhausted coroutine: raise an error, + except when called from gen_close(), which should + always be a silent method. */ + PyErr_SetString( + PyExc_RuntimeError, + "cannot reuse already awaited coroutine"); + } + else if (arg) { + /* `gen` is an exhausted generator: + only return value if called from send(). */ + *presult = Py_NewRef(Py_None); + return PYGEN_RETURN; + } + return PYGEN_ERROR; + } + + assert((frame_state == FRAME_CREATED) || + FRAME_STATE_SUSPENDED(frame_state)); + + if (!_Py_GEN_SET_FRAME_STATE(gen, frame_state, FRAME_EXECUTING)) { + goto retry; + } + + return gen_send_ex2(gen, arg, presult, 0); +} + static PySendResult PyGen_am_send(PyObject *self, PyObject *arg, PyObject **result) { PyGenObject *gen = _PyGen_CAST(self); - return gen_send_ex2(gen, arg, result, 0, 0); + return gen_send_ex(gen, arg, result); } static PyObject * -gen_send_ex(PyGenObject *gen, PyObject *arg, int exc, int closing) +gen_set_stop_iteration(PyGenObject *gen, PyObject *result) { - PyObject *result; - if (gen_send_ex2(gen, arg, &result, exc, closing) == PYGEN_RETURN) { - if (PyAsyncGen_CheckExact(gen)) { - assert(result == Py_None); - PyErr_SetNone(PyExc_StopAsyncIteration); - } - else if (result == Py_None) { - PyErr_SetNone(PyExc_StopIteration); - } - else { - _PyGen_SetStopIterationValue(result); - } - Py_CLEAR(result); + if (PyAsyncGen_CheckExact(gen)) { + assert(result == Py_None); + PyErr_SetNone(PyExc_StopAsyncIteration); } - return result; + else if (result == Py_None) { + PyErr_SetNone(PyExc_StopIteration); + } + else { + _PyGen_SetStopIterationValue(result); + } + Py_DECREF(result); + return NULL; } PyDoc_STRVAR(send_doc, @@ -319,9 +360,14 @@ PyDoc_STRVAR(send_doc, return next yielded value or raise StopIteration."); static PyObject * -gen_send(PyObject *gen, PyObject *arg) +gen_send(PyObject *op, PyObject *arg) { - return gen_send_ex((PyGenObject*)gen, arg, 0, 0); + PyObject *result; + PyGenObject *gen = _PyGen_CAST(op); + if (gen_send_ex(_PyGen_CAST(op), arg, &result) == PYGEN_RETURN) { + return gen_set_stop_iteration(gen, result); + } + return result; } PyDoc_STRVAR(close_doc, @@ -370,42 +416,44 @@ is_resume(_Py_CODEUNIT *instr) ); } -PyObject * -_PyGen_yf(PyGenObject *gen) -{ - if (gen->gi_frame_state == FRAME_SUSPENDED_YIELD_FROM) { - _PyInterpreterFrame *frame = &gen->gi_iframe; - // GH-122390: These asserts are wrong in the presence of ENTER_EXECUTOR! - // assert(is_resume(frame->instr_ptr)); - // assert((frame->instr_ptr->op.arg & RESUME_OPARG_LOCATION_MASK) >= RESUME_AFTER_YIELD_FROM); - return PyStackRef_AsPyObjectNew(_PyFrame_StackPeek(frame)); - } - return NULL; -} - static PyObject * gen_close(PyObject *self, PyObject *args) { PyGenObject *gen = _PyGen_CAST(self); - if (gen->gi_frame_state == FRAME_CREATED) { - gen->gi_frame_state = FRAME_COMPLETED; + int8_t frame_state = FT_ATOMIC_LOAD_INT8_RELAXED(gen->gi_frame_state); +retry: + if (frame_state == FRAME_CREATED) { + if (!_Py_GEN_SET_FRAME_STATE(gen, frame_state, FRAME_COMPLETED)) { + goto retry; + } Py_RETURN_NONE; } - if (FRAME_STATE_FINISHED(gen->gi_frame_state)) { + + if (FRAME_STATE_FINISHED(frame_state)) { Py_RETURN_NONE; } - PyObject *yf = _PyGen_yf(gen); + if (frame_state == FRAME_EXECUTING) { + gen_raise_already_executing_error(gen); + return NULL; + } + + assert(frame_state == FRAME_SUSPENDED_YIELD_FROM || + frame_state == FRAME_SUSPENDED); + + if (!_Py_GEN_SET_FRAME_STATE(gen, frame_state, FRAME_EXECUTING)) { + goto retry; + } + int err = 0; - if (yf) { - PyFrameState state = gen->gi_frame_state; - gen->gi_frame_state = FRAME_EXECUTING; + _PyInterpreterFrame *frame = &gen->gi_iframe; + if (frame_state == FRAME_SUSPENDED_YIELD_FROM) { + PyObject *yf = PyStackRef_AsPyObjectNew(_PyFrame_StackPeek(frame)); err = gen_close_iter(yf); - gen->gi_frame_state = state; Py_DECREF(yf); } - _PyInterpreterFrame *frame = &gen->gi_iframe; + if (is_resume(frame->instr_ptr)) { bool no_unwind_tools = _PyEval_NoToolsForUnwind(_PyThreadState_GET()); /* We can safely ignore the outermost try block @@ -415,7 +463,7 @@ gen_close(PyObject *self, PyObject *args) if (oparg & RESUME_OPARG_DEPTH1_MASK && no_unwind_tools) { // RESUME after YIELD_VALUE and exception depth is 1 assert((oparg & RESUME_OPARG_LOCATION_MASK) != RESUME_AT_FUNC_START); - gen->gi_frame_state = FRAME_COMPLETED; + FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_CLEARED); gen_clear_frame(gen); Py_RETURN_NONE; } @@ -424,8 +472,13 @@ gen_close(PyObject *self, PyObject *args) PyErr_SetNone(PyExc_GeneratorExit); } - PyObject *retval = gen_send_ex(gen, Py_None, 1, 1); - if (retval) { + PyObject *retval; + if (gen_send_ex2(gen, Py_None, &retval, 1) == PYGEN_RETURN) { + // the generator returned a value while closing, return the value here + assert(!PyErr_Occurred()); + return retval; + } + else if (retval) { const char *msg = "generator ignored GeneratorExit"; if (PyCoro_CheckExact(gen)) { msg = "coroutine ignored GeneratorExit"; @@ -442,15 +495,80 @@ gen_close(PyObject *self, PyObject *args) PyErr_Clear(); /* ignore this error */ Py_RETURN_NONE; } + return NULL; +} - /* if the generator returned a value while closing, StopIteration was - * raised in gen_send_ex() above; retrieve and return the value here */ - if (_PyGen_FetchStopIterationValue(&retval) == 0) { - return retval; +// Set an exception for a gen.throw() call. +// Return 0 on success, -1 on failure. +static int +gen_set_exception(PyObject *typ, PyObject *val, PyObject *tb) +{ + /* First, check the traceback argument, replacing None with + NULL. */ + if (tb == Py_None) { + tb = NULL; } - return NULL; + else if (tb != NULL && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "throw() third argument must be a traceback object"); + return -1; + } + + Py_INCREF(typ); + Py_XINCREF(val); + Py_XINCREF(tb); + + if (PyExceptionClass_Check(typ)) + PyErr_NormalizeException(&typ, &val, &tb); + + else if (PyExceptionInstance_Check(typ)) { + /* Raising an instance. The value should be a dummy. */ + if (val && val != Py_None) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto failed_throw; + } + else { + /* Normalize to raise , */ + Py_XSETREF(val, typ); + typ = Py_NewRef(PyExceptionInstance_Class(typ)); + + if (tb == NULL) + /* Returns NULL if there's no traceback */ + tb = PyException_GetTraceback(val); + } + } + else { + /* Not something you can raise. throw() fails. */ + PyErr_Format(PyExc_TypeError, + "exceptions must be classes or instances " + "deriving from BaseException, not %s", + Py_TYPE(typ)->tp_name); + goto failed_throw; + } + + PyErr_Restore(typ, val, tb); + return 0; + +failed_throw: + /* Didn't use our arguments, so restore their original refcounts */ + Py_DECREF(typ); + Py_XDECREF(val); + Py_XDECREF(tb); + return -1; } +static PyObject * +gen_throw_current_exception(PyGenObject *gen) +{ + assert(gen->gi_frame_state == FRAME_EXECUTING); + + PyObject *result; + if (gen_send_ex2(gen, Py_None, &result, 1) == PYGEN_RETURN) { + return gen_set_stop_iteration(gen, result); + } + return result; +} PyDoc_STRVAR(throw_doc, "throw(value)\n\ @@ -465,10 +583,32 @@ static PyObject * _gen_throw(PyGenObject *gen, int close_on_genexit, PyObject *typ, PyObject *val, PyObject *tb) { - PyObject *yf = _PyGen_yf(gen); + int8_t frame_state = FT_ATOMIC_LOAD_INT8_RELAXED(gen->gi_frame_state); +retry: + if (frame_state == FRAME_EXECUTING) { + gen_raise_already_executing_error(gen); + return NULL; + } + + if (FRAME_STATE_FINISHED(frame_state)) { + if (PyCoro_CheckExact(gen)) { + /* `gen` is an exhausted coroutine: raise an error */ + PyErr_SetString( + PyExc_RuntimeError, + "cannot reuse already awaited coroutine"); + return NULL; + } + gen_set_exception(typ, val, tb); + return NULL; + } + + if (!_Py_GEN_SET_FRAME_STATE(gen, frame_state, FRAME_EXECUTING)) { + goto retry; + } - if (yf) { + if (frame_state == FRAME_SUSPENDED_YIELD_FROM) { _PyInterpreterFrame *frame = &gen->gi_iframe; + PyObject *yf = PyStackRef_AsPyObjectNew(_PyFrame_StackPeek(frame)); PyObject *ret; int err; if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit) && @@ -478,13 +618,11 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, We have to allow some awaits to work it through, hence the `close_on_genexit` parameter here. */ - PyFrameState state = gen->gi_frame_state; - gen->gi_frame_state = FRAME_EXECUTING; err = gen_close_iter(yf); - gen->gi_frame_state = state; Py_DECREF(yf); - if (err < 0) - return gen_send_ex(gen, Py_None, 1, 0); + if (err < 0) { + return gen_throw_current_exception(gen); + } goto throw_here; } PyThreadState *tstate = _PyThreadState_GET(); @@ -500,18 +638,17 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, tstate->current_frame = frame; /* Close the generator that we are currently iterating with 'yield from' or awaiting on with 'await'. */ - PyFrameState state = gen->gi_frame_state; - gen->gi_frame_state = FRAME_EXECUTING; ret = _gen_throw((PyGenObject *)yf, close_on_genexit, typ, val, tb); - gen->gi_frame_state = state; tstate->current_frame = prev; frame->previous = NULL; - } else { + } + else { /* `yf` is an iterator or a coroutine-like object. */ PyObject *meth; if (PyObject_GetOptionalAttr(yf, &_Py_ID(throw), &meth) < 0) { Py_DECREF(yf); + FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, frame_state); return NULL; } if (meth == NULL) { @@ -522,75 +659,26 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, _PyInterpreterFrame *prev = tstate->current_frame; frame->previous = prev; tstate->current_frame = frame; - PyFrameState state = gen->gi_frame_state; - gen->gi_frame_state = FRAME_EXECUTING; ret = PyObject_CallFunctionObjArgs(meth, typ, val, tb, NULL); - gen->gi_frame_state = state; tstate->current_frame = prev; frame->previous = NULL; Py_DECREF(meth); } Py_DECREF(yf); if (!ret) { - ret = gen_send_ex(gen, Py_None, 1, 0); + return gen_throw_current_exception(gen); } + FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, frame_state); return ret; } throw_here: - /* First, check the traceback argument, replacing None with - NULL. */ - if (tb == Py_None) { - tb = NULL; - } - else if (tb != NULL && !PyTraceBack_Check(tb)) { - PyErr_SetString(PyExc_TypeError, - "throw() third argument must be a traceback object"); + assert(FT_ATOMIC_LOAD_INT8_RELAXED(gen->gi_frame_state) == FRAME_EXECUTING); + if (gen_set_exception(typ, val, tb) < 0) { + FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, frame_state); return NULL; } - - Py_INCREF(typ); - Py_XINCREF(val); - Py_XINCREF(tb); - - if (PyExceptionClass_Check(typ)) - PyErr_NormalizeException(&typ, &val, &tb); - - else if (PyExceptionInstance_Check(typ)) { - /* Raising an instance. The value should be a dummy. */ - if (val && val != Py_None) { - PyErr_SetString(PyExc_TypeError, - "instance exception may not have a separate value"); - goto failed_throw; - } - else { - /* Normalize to raise , */ - Py_XSETREF(val, typ); - typ = Py_NewRef(PyExceptionInstance_Class(typ)); - - if (tb == NULL) - /* Returns NULL if there's no traceback */ - tb = PyException_GetTraceback(val); - } - } - else { - /* Not something you can raise. throw() fails. */ - PyErr_Format(PyExc_TypeError, - "exceptions must be classes or instances " - "deriving from BaseException, not %s", - Py_TYPE(typ)->tp_name); - goto failed_throw; - } - - PyErr_Restore(typ, val, tb); - return gen_send_ex(gen, Py_None, 1, 0); - -failed_throw: - /* Didn't use our arguments, so restore their original refcounts */ - Py_DECREF(typ); - Py_XDECREF(val); - Py_XDECREF(tb); - return NULL; + return gen_throw_current_exception(gen); } @@ -632,7 +720,7 @@ gen_iternext(PyObject *self) PyGenObject *gen = _PyGen_CAST(self); PyObject *result; - if (gen_send_ex2(gen, NULL, &result, 0, 0) == PYGEN_RETURN) { + if (gen_send_ex(gen, NULL, &result) == PYGEN_RETURN) { if (result != Py_None) { _PyGen_SetStopIterationValue(result); } @@ -756,13 +844,15 @@ gen_set_qualname(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) } static PyObject * -gen_getyieldfrom(PyObject *gen, void *Py_UNUSED(ignored)) +gen_getyieldfrom(PyObject *self, void *Py_UNUSED(ignored)) { - PyObject *yf = _PyGen_yf(_PyGen_CAST(gen)); - if (yf == NULL) { + PyGenObject *gen = _PyGen_CAST(self); + int8_t frame_state = FT_ATOMIC_LOAD_INT8_RELAXED(gen->gi_frame_state); + if (frame_state != FRAME_SUSPENDED_YIELD_FROM) { Py_RETURN_NONE; } - return yf; + // TODO: still not thread-safe with free threading + return PyStackRef_AsPyObjectNew(_PyFrame_StackPeek(&gen->gi_iframe)); } @@ -770,17 +860,16 @@ static PyObject * gen_getrunning(PyObject *self, void *Py_UNUSED(ignored)) { PyGenObject *gen = _PyGen_CAST(self); - if (gen->gi_frame_state == FRAME_EXECUTING) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; + int8_t frame_state = FT_ATOMIC_LOAD_INT8_RELAXED(gen->gi_frame_state); + return frame_state == FRAME_EXECUTING ? Py_True : Py_False; } static PyObject * gen_getsuspended(PyObject *self, void *Py_UNUSED(ignored)) { PyGenObject *gen = _PyGen_CAST(self); - return PyBool_FromLong(FRAME_STATE_SUSPENDED(gen->gi_frame_state)); + int8_t frame_state = FT_ATOMIC_LOAD_INT8_RELAXED(gen->gi_frame_state); + return FRAME_STATE_SUSPENDED(frame_state) ? Py_True : Py_False; } static PyObject * @@ -789,9 +878,11 @@ _gen_getframe(PyGenObject *gen, const char *const name) if (PySys_Audit("object.__getattr__", "Os", gen, name) < 0) { return NULL; } - if (FRAME_STATE_FINISHED(gen->gi_frame_state)) { + int8_t frame_state = FT_ATOMIC_LOAD_INT8_RELAXED(gen->gi_frame_state); + if (FRAME_STATE_FINISHED(frame_state)) { Py_RETURN_NONE; } + // TODO: still not thread-safe with free threading return _Py_XNewRef((PyObject *)_PyFrame_GetFrameObject(&gen->gi_iframe)); } @@ -1134,35 +1225,6 @@ coro_await(PyObject *coro) return (PyObject *)cw; } -static PyObject * -coro_get_cr_await(PyObject *coro, void *Py_UNUSED(ignored)) -{ - PyObject *yf = _PyGen_yf((PyGenObject *) coro); - if (yf == NULL) - Py_RETURN_NONE; - return yf; -} - -static PyObject * -cr_getsuspended(PyObject *self, void *Py_UNUSED(ignored)) -{ - PyCoroObject *coro = _PyCoroObject_CAST(self); - if (FRAME_STATE_SUSPENDED(coro->cr_frame_state)) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; -} - -static PyObject * -cr_getrunning(PyObject *self, void *Py_UNUSED(ignored)) -{ - PyCoroObject *coro = _PyCoroObject_CAST(self); - if (coro->cr_frame_state == FRAME_EXECUTING) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; -} - static PyObject * cr_getframe(PyObject *coro, void *Py_UNUSED(ignored)) { @@ -1180,12 +1242,12 @@ static PyGetSetDef coro_getsetlist[] = { PyDoc_STR("name of the coroutine")}, {"__qualname__", gen_get_qualname, gen_set_qualname, PyDoc_STR("qualified name of the coroutine")}, - {"cr_await", coro_get_cr_await, NULL, + {"cr_await", gen_getyieldfrom, NULL, PyDoc_STR("object being awaited on, or None")}, - {"cr_running", cr_getrunning, NULL, NULL}, + {"cr_running", gen_getrunning, NULL, NULL}, {"cr_frame", cr_getframe, NULL, NULL}, {"cr_code", cr_getcode, NULL, NULL}, - {"cr_suspended", cr_getsuspended, NULL, NULL}, + {"cr_suspended", gen_getsuspended, NULL, NULL}, {NULL} /* Sentinel */ }; @@ -1601,26 +1663,16 @@ ag_getcode(PyObject *gen, void *Py_UNUSED(ignored)) return _gen_getcode((PyGenObject*)gen, "ag_code"); } -static PyObject * -ag_getsuspended(PyObject *self, void *Py_UNUSED(ignored)) -{ - PyAsyncGenObject *ag = _PyAsyncGenObject_CAST(self); - if (FRAME_STATE_SUSPENDED(ag->ag_frame_state)) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; -} - static PyGetSetDef async_gen_getsetlist[] = { {"__name__", gen_get_name, gen_set_name, PyDoc_STR("name of the async generator")}, {"__qualname__", gen_get_qualname, gen_set_qualname, PyDoc_STR("qualified name of the async generator")}, - {"ag_await", coro_get_cr_await, NULL, + {"ag_await", gen_getyieldfrom, NULL, PyDoc_STR("object being awaited on, or None")}, {"ag_frame", ag_getframe, NULL, NULL}, {"ag_code", ag_getcode, NULL, NULL}, - {"ag_suspended", ag_getsuspended, NULL, NULL}, + {"ag_suspended", gen_getsuspended, NULL, NULL}, {NULL} /* Sentinel */ }; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index cb39a23d951077..b155013d24f4e6 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1338,14 +1338,13 @@ dummy_func( assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if ((tstate->interp->eval_frame == NULL) && (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && - ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) + _PyGen_SetExecuting((PyGenObject *)receiver_o)) { PyGenObject *gen = (PyGenObject *)receiver_o; _PyInterpreterFrame *gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v)); DEAD(v); SYNC_SP(); - gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; assert(INSTRUCTION_SIZE + oparg <= UINT16_MAX); @@ -1386,12 +1385,11 @@ dummy_func( op(_SEND_GEN_FRAME, (receiver, v -- receiver, gen_frame)) { PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING); + DEOPT_IF(!_PyGen_SetExecuting((PyGenObject *)gen)); STAT_INC(SEND, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); DEAD(v); - gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; assert(INSTRUCTION_SIZE + oparg <= UINT16_MAX); @@ -1415,7 +1413,6 @@ dummy_func( PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; _PyStackRef temp = retval; DEAD(retval); SAVE_STACK(); @@ -1425,6 +1422,10 @@ dummy_func( _PyInterpreterFrame *gen_frame = frame; frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; + FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); +#ifdef Py_GIL_DISABLED + ((_PyThreadStateImpl *)tstate)->gen_last_frame_state = FRAME_SUSPENDED + oparg; +#endif /* We don't know which of these is relevant here, so keep them equal */ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE @@ -3423,18 +3424,10 @@ dummy_func( op(_FOR_ITER_GEN_FRAME, (iter, null -- iter, null, gen_frame)) { PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type); -#ifdef Py_GIL_DISABLED - // Since generators can't be used by multiple threads anyway we - // don't need to deopt here, but this lets us work on making - // generators thread-safe without necessarily having to - // specialize them thread-safely as well. - DEOPT_IF(!_PyObject_IsUniquelyReferenced((PyObject *)gen)); -#endif - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING); + DEOPT_IF(!_PyGen_SetExecuting((PyGenObject *)gen)); STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; pushed_frame->previous = frame; diff --git a/Python/ceval.c b/Python/ceval.c index 37679d4cd183c7..0000b5c2f96f6e 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2304,7 +2304,10 @@ clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) { assert(frame->owner == FRAME_OWNED_BY_GENERATOR); PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - gen->gi_frame_state = FRAME_CLEARED; + FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_CLEARED); +#ifdef Py_GIL_DISABLED + ((_PyThreadStateImpl *)tstate)->gen_last_frame_state = FRAME_CLEARED; +#endif assert(tstate->exc_info == &gen->gi_exc_state); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -3979,15 +3982,13 @@ _PyEval_GetAwaitable(PyObject *iterable, int oparg) Py_TYPE(iterable), oparg); } else if (PyCoro_CheckExact(iter)) { - PyObject *yf = _PyGen_yf((PyGenObject*)iter); - if (yf != NULL) { - /* `iter` is a coroutine object that is being - awaited, `yf` is a pointer to the current awaitable - being awaited on. */ - Py_DECREF(yf); + PyCoroObject *coro = (PyCoroObject *)iter; + int8_t frame_state = FT_ATOMIC_LOAD_INT8_RELAXED(coro->cr_frame_state); + if (frame_state == FRAME_SUSPENDED_YIELD_FROM) { + /* `iter` is a coroutine object that is being awaited. */ Py_CLEAR(iter); _PyErr_SetString(PyThreadState_GET(), PyExc_RuntimeError, - "coroutine is being awaited already"); + "coroutine is being awaited already"); } } return iter; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 598fea4dfbf692..f311ff041d1fa4 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -6089,7 +6089,7 @@ SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - if (gen->gi_frame_state >= FRAME_EXECUTING) { + if (!_PyGen_SetExecuting((PyGenObject *)gen)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); @@ -6097,7 +6097,6 @@ STAT_INC(SEND, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); - gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; assert( 2u + oparg <= UINT16_MAX); @@ -6125,7 +6124,6 @@ PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; _PyStackRef temp = retval; _PyFrame_SetStackPointer(frame, stack_pointer); tstate->exc_info = gen->gi_exc_state.previous_item; @@ -6134,6 +6132,11 @@ _PyInterpreterFrame *gen_frame = frame; frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; + FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); + #ifdef Py_GIL_DISABLED + ((_PyThreadStateImpl *)tstate)->gen_last_frame_state = FRAME_SUSPENDED + oparg; + #endif + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || @@ -10813,30 +10816,95 @@ break; } - case _FOR_ITER_GEN_FRAME_r23: { - CHECK_CURRENT_CACHED_VALUES(2); + case _FOR_ITER_GEN_FRAME_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef iter; + _PyStackRef gen_frame; + oparg = CURRENT_OPARG(); + iter = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(gen) != &PyGen_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (!_PyGen_SetExecuting((PyGenObject *)gen)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(FOR_ITER, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_None); + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + pushed_frame->previous = frame; + frame->return_offset = (uint16_t)( 2u + oparg); + gen_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache2 = gen_frame; + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _FOR_ITER_GEN_FRAME_r13: { + CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef iter; _PyStackRef gen_frame; _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); - iter = _stack_item_0; + iter = stack_pointer[-1]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(gen) != &PyGen_Type) { UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(2); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } - #ifdef Py_GIL_DISABLED + if (!_PyGen_SetExecuting((PyGenObject *)gen)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(FOR_ITER, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_None); + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + pushed_frame->previous = frame; + frame->return_offset = (uint16_t)( 2u + oparg); + gen_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache2 = gen_frame; + _tos_cache1 = _stack_item_0; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } - if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { + case _FOR_ITER_GEN_FRAME_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef iter; + _PyStackRef gen_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + iter = _stack_item_0; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(gen) != &PyGen_Type) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - #endif - if (gen->gi_frame_state >= FRAME_EXECUTING) { + if (!_PyGen_SetExecuting((PyGenObject *)gen)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); @@ -10844,7 +10912,6 @@ STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; pushed_frame->previous = frame; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a0e6fd2d7cba27..7c1828529b202e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5380,14 +5380,7 @@ assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); - } - #endif - if (gen->gi_frame_state >= FRAME_EXECUTING) { + if (!_PyGen_SetExecuting((PyGenObject *)gen)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); @@ -5395,7 +5388,6 @@ STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; pushed_frame->previous = frame; @@ -7159,7 +7151,6 @@ PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; _PyStackRef temp = retval; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -7170,6 +7161,11 @@ _PyInterpreterFrame *gen_frame = frame; frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; + FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); + #ifdef Py_GIL_DISABLED + ((_PyThreadStateImpl *)tstate)->gen_last_frame_state = FRAME_SUSPENDED + oparg; + #endif + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || @@ -10126,14 +10122,13 @@ assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if ((tstate->interp->eval_frame == NULL) && (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && - ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) + _PyGen_SetExecuting((PyGenObject *)receiver_o)) { PyGenObject *gen = (PyGenObject *)receiver_o; _PyInterpreterFrame *gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v)); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; assert( 2u + oparg <= UINT16_MAX); @@ -10226,7 +10221,7 @@ assert(_PyOpcode_Deopt[opcode] == (SEND)); JUMP_TO_PREDICTED(SEND); } - if (gen->gi_frame_state >= FRAME_EXECUTING) { + if (!_PyGen_SetExecuting((PyGenObject *)gen)) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); JUMP_TO_PREDICTED(SEND); @@ -10234,7 +10229,6 @@ STAT_INC(SEND, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); - gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; assert( 2u + oparg <= UINT16_MAX); @@ -11807,7 +11801,6 @@ PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; _PyStackRef temp = retval; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -11818,6 +11811,11 @@ _PyInterpreterFrame *gen_frame = frame; frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; + FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); + #ifdef Py_GIL_DISABLED + ((_PyThreadStateImpl *)tstate)->gen_last_frame_state = FRAME_SUSPENDED + oparg; + #endif + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index f6575a93e47a31..9e7bd41e02a578 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -643,6 +643,7 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "_PyFrame_StackPush", "_PyFunction_SetVersion", "_PyGen_GetGeneratorFromFrame", + "_PyGen_SetExecuting", "_PyInterpreterState_GET", "_PyList_AppendTakeRef", "_PyList_ITEMS", From 77fee3579c3b26e705603a1430e726308727729f Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 12 Dec 2025 11:43:43 -0500 Subject: [PATCH 2/9] Add some tests --- .../test_free_threading/test_generators.py | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/Lib/test/test_free_threading/test_generators.py b/Lib/test/test_free_threading/test_generators.py index d01675eb38b370..b0fc40c3ec6964 100644 --- a/Lib/test/test_free_threading/test_generators.py +++ b/Lib/test/test_free_threading/test_generators.py @@ -1,5 +1,6 @@ import concurrent.futures import unittest +import threading from threading import Barrier from unittest import TestCase import random @@ -49,3 +50,74 @@ def test_concurrent_write(self): self.concurrent_write_with_func(func=set_gen_name) with self.subTest(func=set_gen_qualname): self.concurrent_write_with_func(func=set_gen_qualname) + + def test_concurrent_send(self): + def gen(): + yield 1 + yield 2 + yield 3 + yield 4 + yield 5 + + def run_test(drive_generator): + g = gen() + values = [] + threading_helper.run_concurrently(drive_generator, self.NUM_THREADS, args=(g, values,)) + self.assertEqual(sorted(values), [1, 2, 3, 4, 5]) + + def call_next(g, values): + while True: + try: + values.append(next(g)) + except ValueError: + continue + except StopIteration: + break + + with self.subTest(method='next'): + run_test(call_next) + + def call_send(g, values): + while True: + try: + values.append(g.send(None)) + except ValueError: + continue + except StopIteration: + break + + with self.subTest(method='send'): + run_test(call_send) + + def for_iter_gen(g, values): + while True: + try: + for value in g: + values.append(value) + else: + break + except ValueError: + continue + + with self.subTest(method='for'): + run_test(for_iter_gen) + + def test_concurrent_close(self): + def gen(): + for i in range(10): + yield i + time.sleep(0.001) + + def drive_generator(g): + while True: + try: + for value in g: + if value == 5: + g.close() + else: + return + except ValueError as e: + self.assertEqual(e.args[0], "generator already executing") + + g = gen() + threading_helper.run_concurrently(drive_generator, self.NUM_THREADS, args=(g,)) From ac3974df0bef87b3471c6226d5d7f22dd071af32 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 12 Dec 2025 11:45:32 -0500 Subject: [PATCH 3/9] Remove unused import --- Lib/test/test_free_threading/test_generators.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_free_threading/test_generators.py b/Lib/test/test_free_threading/test_generators.py index b0fc40c3ec6964..11f59301bcd51d 100644 --- a/Lib/test/test_free_threading/test_generators.py +++ b/Lib/test/test_free_threading/test_generators.py @@ -1,6 +1,5 @@ import concurrent.futures import unittest -import threading from threading import Barrier from unittest import TestCase import random From b285594849a5ed197bed1edba4ed563749f7270e Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 15 Dec 2025 11:43:48 -0500 Subject: [PATCH 4/9] Update Objects/genobject.c Co-authored-by: Kumar Aditya --- Objects/genobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/genobject.c b/Objects/genobject.c index de4206e9a1330b..d99c8c673a4a2e 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -364,7 +364,7 @@ gen_send(PyObject *op, PyObject *arg) { PyObject *result; PyGenObject *gen = _PyGen_CAST(op); - if (gen_send_ex(_PyGen_CAST(op), arg, &result) == PYGEN_RETURN) { + if (gen_send_ex(gen, arg, &result) == PYGEN_RETURN) { return gen_set_stop_iteration(gen, result); } return result; From 9c1c0fe9ccf24d2f3afedc7415145afcfbf05460 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 15 Dec 2025 11:44:00 -0500 Subject: [PATCH 5/9] Update Objects/genobject.c Co-authored-by: Kumar Aditya --- Objects/genobject.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/genobject.c b/Objects/genobject.c index d99c8c673a4a2e..5f02ab97d2f0fc 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -518,9 +518,9 @@ gen_set_exception(PyObject *typ, PyObject *val, PyObject *tb) Py_XINCREF(val); Py_XINCREF(tb); - if (PyExceptionClass_Check(typ)) + if (PyExceptionClass_Check(typ)) { PyErr_NormalizeException(&typ, &val, &tb); - + } else if (PyExceptionInstance_Check(typ)) { /* Raising an instance. The value should be a dummy. */ if (val && val != Py_None) { From 3876dd0d4b780812f455a5a6121e296e23868064 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 15 Dec 2025 16:45:33 +0000 Subject: [PATCH 6/9] Changes from review --- Include/internal/pycore_genobject.h | 24 ------------------- Objects/genobject.c | 2 +- Python/bytecodes.c | 11 ++++----- Python/ceval.c | 5 +--- Python/ceval_macros.h | 36 +++++++++++++++++++++++++++++ Python/executor_cases.c.h | 14 ++++------- Python/generated_cases.c.h | 18 ++++----------- Tools/cases_generator/analyzer.py | 3 ++- 8 files changed, 54 insertions(+), 59 deletions(-) diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index d73bef50c5a9ba..b2b5520047e6fe 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -34,30 +34,6 @@ PyGenObject *_PyGen_GetGeneratorFromFrame(_PyInterpreterFrame *frame) return (PyGenObject *)(((char *)frame) - offset_in_gen); } -// Mark the generator as executing. Returns true if the state was changed, -// false if it was already executing or finished. -static inline bool -_PyGen_SetExecuting(PyGenObject *gen) -{ -#ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { - int8_t frame_state = _Py_atomic_load_int8_relaxed(&gen->gi_frame_state); - while (frame_state < FRAME_EXECUTING) { - if (_Py_atomic_compare_exchange_int8(&gen->gi_frame_state, - &frame_state, - FRAME_EXECUTING)) { - return true; - } - } - } -#endif - if (gen->gi_frame_state < FRAME_EXECUTING) { - gen->gi_frame_state = FRAME_EXECUTING; - return true; - } - return false; -} - extern void _PyGen_Finalize(PyObject *self); // Export for '_asyncio' shared extension diff --git a/Objects/genobject.c b/Objects/genobject.c index 5f02ab97d2f0fc..e9aca8fd4d0834 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -234,7 +234,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, int exc) _PyErr_ChainStackItem(); } -#if defined(Py_GIL_DISABLED) && defined(Py_DEBUG) +#if defined(Py_GIL_DISABLED) ((_PyThreadStateImpl *)tstate)->gen_last_frame_state = FRAME_EXECUTING; #endif diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5a4c2c978f3710..608ed3541a35ed 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1338,7 +1338,7 @@ dummy_func( assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if ((tstate->interp->eval_frame == NULL) && (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && - _PyGen_SetExecuting((PyGenObject *)receiver_o)) + gen_try_set_executing((PyGenObject *)receiver_o)) { PyGenObject *gen = (PyGenObject *)receiver_o; _PyInterpreterFrame *gen_frame = &gen->gi_iframe; @@ -1385,7 +1385,7 @@ dummy_func( op(_SEND_GEN_FRAME, (receiver, v -- receiver, gen_frame)) { PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type); - DEOPT_IF(!_PyGen_SetExecuting((PyGenObject *)gen)); + DEOPT_IF(!gen_try_set_executing((PyGenObject *)gen)); STAT_INC(SEND, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); @@ -1422,10 +1422,7 @@ dummy_func( _PyInterpreterFrame *gen_frame = frame; frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; - FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); -#ifdef Py_GIL_DISABLED - ((_PyThreadStateImpl *)tstate)->gen_last_frame_state = FRAME_SUSPENDED + oparg; -#endif + gen_set_frame_state(gen, tstate, FRAME_SUSPENDED + oparg); /* We don't know which of these is relevant here, so keep them equal */ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE @@ -3424,7 +3421,7 @@ dummy_func( op(_FOR_ITER_GEN_FRAME, (iter, null -- iter, null, gen_frame)) { PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); DEOPT_IF(Py_TYPE(gen) != &PyGen_Type); - DEOPT_IF(!_PyGen_SetExecuting((PyGenObject *)gen)); + DEOPT_IF(!gen_try_set_executing((PyGenObject *)gen)); STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); diff --git a/Python/ceval.c b/Python/ceval.c index 0000b5c2f96f6e..d4c2d37ef35dbc 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2304,10 +2304,7 @@ clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) { assert(frame->owner == FRAME_OWNED_BY_GENERATOR); PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_CLEARED); -#ifdef Py_GIL_DISABLED - ((_PyThreadStateImpl *)tstate)->gen_last_frame_state = FRAME_CLEARED; -#endif + gen_set_frame_state(gen, tstate, FRAME_CLEARED); assert(tstate->exc_info == &gen->gi_exc_state); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 26312d0ea5aa5c..31dfa65a3f0912 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -495,3 +495,39 @@ check_periodics(PyThreadState *tstate) { return 0; } + + +// Mark the generator as executing. Returns true if the state was changed, +// false if it was already executing or finished. +static inline bool +gen_try_set_executing(PyGenObject *gen) +{ +#ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { + int8_t frame_state = _Py_atomic_load_int8_relaxed(&gen->gi_frame_state); + while (frame_state < FRAME_EXECUTING) { + if (_Py_atomic_compare_exchange_int8(&gen->gi_frame_state, + &frame_state, + FRAME_EXECUTING)) { + return true; + } + } + } +#endif + // Use faster non-atomic modifications in the GIL-enabled build and when + // the object is uniquely referenced in the free-threaded build. + if (gen->gi_frame_state < FRAME_EXECUTING) { + gen->gi_frame_state = FRAME_EXECUTING; + return true; + } + return false; +} + +static inline void +gen_set_frame_state(PyGenObject *gen, PyThreadState *tstate, int8_t frame_state) +{ + FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, frame_state); +#ifdef Py_GIL_DISABLED + ((_PyThreadStateImpl *)tstate)->gen_last_frame_state = frame_state; +#endif +} diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index d5ef0cab2e3d1d..ed68ee581697b5 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -6089,7 +6089,7 @@ SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - if (!_PyGen_SetExecuting((PyGenObject *)gen)) { + if (!gen_try_set_executing((PyGenObject *)gen)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); @@ -6132,11 +6132,7 @@ _PyInterpreterFrame *gen_frame = frame; frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; - FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); - #ifdef Py_GIL_DISABLED - ((_PyThreadStateImpl *)tstate)->gen_last_frame_state = FRAME_SUSPENDED + oparg; - #endif - + gen_set_frame_state(gen, tstate, FRAME_SUSPENDED + oparg); assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || @@ -10829,7 +10825,7 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (!_PyGen_SetExecuting((PyGenObject *)gen)) { + if (!gen_try_set_executing((PyGenObject *)gen)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); @@ -10866,7 +10862,7 @@ SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } - if (!_PyGen_SetExecuting((PyGenObject *)gen)) { + if (!gen_try_set_executing((PyGenObject *)gen)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); @@ -10904,7 +10900,7 @@ SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - if (!_PyGen_SetExecuting((PyGenObject *)gen)) { + if (!gen_try_set_executing((PyGenObject *)gen)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d37845d2e9f642..5fa3859f4a8298 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5375,7 +5375,7 @@ assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } - if (!_PyGen_SetExecuting((PyGenObject *)gen)) { + if (!gen_try_set_executing((PyGenObject *)gen)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); @@ -7156,11 +7156,7 @@ _PyInterpreterFrame *gen_frame = frame; frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; - FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); - #ifdef Py_GIL_DISABLED - ((_PyThreadStateImpl *)tstate)->gen_last_frame_state = FRAME_SUSPENDED + oparg; - #endif - + gen_set_frame_state(gen, tstate, FRAME_SUSPENDED + oparg); assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || @@ -10117,7 +10113,7 @@ assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if ((tstate->interp->eval_frame == NULL) && (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && - _PyGen_SetExecuting((PyGenObject *)receiver_o)) + gen_try_set_executing((PyGenObject *)receiver_o)) { PyGenObject *gen = (PyGenObject *)receiver_o; _PyInterpreterFrame *gen_frame = &gen->gi_iframe; @@ -10216,7 +10212,7 @@ assert(_PyOpcode_Deopt[opcode] == (SEND)); JUMP_TO_PREDICTED(SEND); } - if (!_PyGen_SetExecuting((PyGenObject *)gen)) { + if (!gen_try_set_executing((PyGenObject *)gen)) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); JUMP_TO_PREDICTED(SEND); @@ -11806,11 +11802,7 @@ _PyInterpreterFrame *gen_frame = frame; frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; - FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); - #ifdef Py_GIL_DISABLED - ((_PyThreadStateImpl *)tstate)->gen_last_frame_state = FRAME_SUSPENDED + oparg; - #endif - + gen_set_frame_state(gen, tstate, FRAME_SUSPENDED + oparg); assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 9e7bd41e02a578..a35cfe5e9a5c38 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -643,7 +643,8 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "_PyFrame_StackPush", "_PyFunction_SetVersion", "_PyGen_GetGeneratorFromFrame", - "_PyGen_SetExecuting", + "gen_try_set_executing", + "gen_set_frame_state", "_PyInterpreterState_GET", "_PyList_AppendTakeRef", "_PyList_ITEMS", From 46a8e84b05968adc0093bbc796a0a64d35649dd1 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 15 Dec 2025 16:54:15 +0000 Subject: [PATCH 7/9] Remove extra whitespace --- Python/ceval_macros.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 31dfa65a3f0912..acd7dd430b8ea8 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -495,8 +495,6 @@ check_periodics(PyThreadState *tstate) { return 0; } - - // Mark the generator as executing. Returns true if the state was changed, // false if it was already executing or finished. static inline bool From 64155ab438951cdb008196b4d9f7db5cd867d84e Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 16 Dec 2025 12:04:44 -0500 Subject: [PATCH 8/9] Rename _Py_GEN_SET_FRAME_STATE to _Py_GEN_TRY_SET_FRAME_STATE --- Objects/genobject.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Objects/genobject.c b/Objects/genobject.c index e9aca8fd4d0834..3179b7ddf5d77f 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -37,10 +37,10 @@ static PyObject* async_gen_athrow_new(PyAsyncGenObject *, PyObject *); _Py_CAST(PyAsyncGenObject*, (op)) #ifdef Py_GIL_DISABLED -# define _Py_GEN_SET_FRAME_STATE(gen, expected, state) \ +# define _Py_GEN_TRY_SET_FRAME_STATE(gen, expected, state) \ _Py_atomic_compare_exchange_int8(&(gen)->gi_frame_state, &expected, (state)) #else -# define _Py_GEN_SET_FRAME_STATE(gen, expected, state) \ +# define _Py_GEN_TRY_SET_FRAME_STATE(gen, expected, state) \ ((gen)->gi_frame_state = (state), true) #endif @@ -324,7 +324,7 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, PyObject **presult) assert((frame_state == FRAME_CREATED) || FRAME_STATE_SUSPENDED(frame_state)); - if (!_Py_GEN_SET_FRAME_STATE(gen, frame_state, FRAME_EXECUTING)) { + if (!_Py_GEN_TRY_SET_FRAME_STATE(gen, frame_state, FRAME_EXECUTING)) { goto retry; } @@ -424,7 +424,7 @@ gen_close(PyObject *self, PyObject *args) int8_t frame_state = FT_ATOMIC_LOAD_INT8_RELAXED(gen->gi_frame_state); retry: if (frame_state == FRAME_CREATED) { - if (!_Py_GEN_SET_FRAME_STATE(gen, frame_state, FRAME_COMPLETED)) { + if (!_Py_GEN_TRY_SET_FRAME_STATE(gen, frame_state, FRAME_COMPLETED)) { goto retry; } Py_RETURN_NONE; @@ -442,7 +442,7 @@ gen_close(PyObject *self, PyObject *args) assert(frame_state == FRAME_SUSPENDED_YIELD_FROM || frame_state == FRAME_SUSPENDED); - if (!_Py_GEN_SET_FRAME_STATE(gen, frame_state, FRAME_EXECUTING)) { + if (!_Py_GEN_TRY_SET_FRAME_STATE(gen, frame_state, FRAME_EXECUTING)) { goto retry; } @@ -602,7 +602,7 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, return NULL; } - if (!_Py_GEN_SET_FRAME_STATE(gen, frame_state, FRAME_EXECUTING)) { + if (!_Py_GEN_TRY_SET_FRAME_STATE(gen, frame_state, FRAME_EXECUTING)) { goto retry; } From f135b6434cd827fff5a13101e79890a4d216989e Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 16 Dec 2025 12:15:37 -0500 Subject: [PATCH 9/9] Switch from goto to do/while --- Objects/genobject.c | 142 +++++++++++++++++++++----------------------- 1 file changed, 69 insertions(+), 73 deletions(-) diff --git a/Objects/genobject.c b/Objects/genobject.c index 3179b7ddf5d77f..cd883e9217f019 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -153,6 +153,7 @@ _PyGen_Finalize(PyObject *self) static void gen_clear_frame(PyGenObject *gen) { + assert(gen->gi_frame_state == FRAME_CLEARED); _PyInterpreterFrame *frame = &gen->gi_iframe; frame->previous = NULL; _PyFrame_ClearExceptCode(frame); @@ -285,48 +286,45 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, PyObject **presult) { *presult = NULL; int8_t frame_state = FT_ATOMIC_LOAD_INT8_RELAXED(gen->gi_frame_state); -retry: - if (frame_state == FRAME_CREATED && arg && arg != Py_None) { - const char *msg = "can't send non-None value to a " - "just-started generator"; - if (PyCoro_CheckExact(gen)) { - msg = NON_INIT_CORO_MSG; - } - else if (PyAsyncGen_CheckExact(gen)) { - msg = "can't send non-None value to a " - "just-started async generator"; + do { + if (frame_state == FRAME_CREATED && arg && arg != Py_None) { + const char *msg = "can't send non-None value to a " + "just-started generator"; + if (PyCoro_CheckExact(gen)) { + msg = NON_INIT_CORO_MSG; + } + else if (PyAsyncGen_CheckExact(gen)) { + msg = "can't send non-None value to a " + "just-started async generator"; + } + PyErr_SetString(PyExc_TypeError, msg); + return PYGEN_ERROR; } - PyErr_SetString(PyExc_TypeError, msg); - return PYGEN_ERROR; - } - if (frame_state == FRAME_EXECUTING) { - gen_raise_already_executing_error(gen); - return PYGEN_ERROR; - } - if (FRAME_STATE_FINISHED(frame_state)) { - if (PyCoro_CheckExact(gen)) { - /* `gen` is an exhausted coroutine: raise an error, - except when called from gen_close(), which should - always be a silent method. */ - PyErr_SetString( - PyExc_RuntimeError, - "cannot reuse already awaited coroutine"); + if (frame_state == FRAME_EXECUTING) { + gen_raise_already_executing_error(gen); + return PYGEN_ERROR; } - else if (arg) { - /* `gen` is an exhausted generator: - only return value if called from send(). */ - *presult = Py_NewRef(Py_None); - return PYGEN_RETURN; + if (FRAME_STATE_FINISHED(frame_state)) { + if (PyCoro_CheckExact(gen)) { + /* `gen` is an exhausted coroutine: raise an error, + except when called from gen_close(), which should + always be a silent method. */ + PyErr_SetString( + PyExc_RuntimeError, + "cannot reuse already awaited coroutine"); + } + else if (arg) { + /* `gen` is an exhausted generator: + only return value if called from send(). */ + *presult = Py_NewRef(Py_None); + return PYGEN_RETURN; + } + return PYGEN_ERROR; } - return PYGEN_ERROR; - } - assert((frame_state == FRAME_CREATED) || - FRAME_STATE_SUSPENDED(frame_state)); - - if (!_Py_GEN_TRY_SET_FRAME_STATE(gen, frame_state, FRAME_EXECUTING)) { - goto retry; - } + assert((frame_state == FRAME_CREATED) || + FRAME_STATE_SUSPENDED(frame_state)); + } while (!_Py_GEN_TRY_SET_FRAME_STATE(gen, frame_state, FRAME_EXECUTING)); return gen_send_ex2(gen, arg, presult, 0); } @@ -422,29 +420,27 @@ gen_close(PyObject *self, PyObject *args) PyGenObject *gen = _PyGen_CAST(self); int8_t frame_state = FT_ATOMIC_LOAD_INT8_RELAXED(gen->gi_frame_state); -retry: - if (frame_state == FRAME_CREATED) { - if (!_Py_GEN_TRY_SET_FRAME_STATE(gen, frame_state, FRAME_COMPLETED)) { - goto retry; + do { + if (frame_state == FRAME_CREATED) { + if (!_Py_GEN_TRY_SET_FRAME_STATE(gen, frame_state, FRAME_COMPLETED)) { + continue; + } + Py_RETURN_NONE; } - Py_RETURN_NONE; - } - if (FRAME_STATE_FINISHED(frame_state)) { - Py_RETURN_NONE; - } + if (FRAME_STATE_FINISHED(frame_state)) { + Py_RETURN_NONE; + } - if (frame_state == FRAME_EXECUTING) { - gen_raise_already_executing_error(gen); - return NULL; - } + if (frame_state == FRAME_EXECUTING) { + gen_raise_already_executing_error(gen); + return NULL; + } - assert(frame_state == FRAME_SUSPENDED_YIELD_FROM || - frame_state == FRAME_SUSPENDED); + assert(frame_state == FRAME_SUSPENDED_YIELD_FROM || + frame_state == FRAME_SUSPENDED); - if (!_Py_GEN_TRY_SET_FRAME_STATE(gen, frame_state, FRAME_EXECUTING)) { - goto retry; - } + } while (!_Py_GEN_TRY_SET_FRAME_STATE(gen, frame_state, FRAME_EXECUTING)); int err = 0; _PyInterpreterFrame *frame = &gen->gi_iframe; @@ -584,27 +580,27 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, PyObject *typ, PyObject *val, PyObject *tb) { int8_t frame_state = FT_ATOMIC_LOAD_INT8_RELAXED(gen->gi_frame_state); -retry: - if (frame_state == FRAME_EXECUTING) { - gen_raise_already_executing_error(gen); - return NULL; - } + do { + if (frame_state == FRAME_EXECUTING) { + gen_raise_already_executing_error(gen); + return NULL; + } - if (FRAME_STATE_FINISHED(frame_state)) { - if (PyCoro_CheckExact(gen)) { - /* `gen` is an exhausted coroutine: raise an error */ - PyErr_SetString( - PyExc_RuntimeError, - "cannot reuse already awaited coroutine"); + if (FRAME_STATE_FINISHED(frame_state)) { + if (PyCoro_CheckExact(gen)) { + /* `gen` is an exhausted coroutine: raise an error */ + PyErr_SetString( + PyExc_RuntimeError, + "cannot reuse already awaited coroutine"); + return NULL; + } + gen_set_exception(typ, val, tb); return NULL; } - gen_set_exception(typ, val, tb); - return NULL; - } - if (!_Py_GEN_TRY_SET_FRAME_STATE(gen, frame_state, FRAME_EXECUTING)) { - goto retry; - } + assert((frame_state == FRAME_CREATED) || + FRAME_STATE_SUSPENDED(frame_state)); + } while (!_Py_GEN_TRY_SET_FRAME_STATE(gen, frame_state, FRAME_EXECUTING)); if (frame_state == FRAME_SUSPENDED_YIELD_FROM) { _PyInterpreterFrame *frame = &gen->gi_iframe;