From 10a16c5d1b8aa7d7c3cea989c65d5ffa583eb02d Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 1 Apr 2025 18:04:54 -0700 Subject: [PATCH 1/5] Add guards for dicts, lists, sets, and tuples --- Include/internal/pycore_opcode_metadata.h | 38 +- Include/internal/pycore_uop_ids.h | 387 +++++++------ Include/internal/pycore_uop_metadata.h | 38 +- Python/bytecodes.c | 116 +++- Python/executor_cases.c.h | 146 +++-- Python/generated_cases.c.h | 667 ++++++++++++---------- Python/optimizer_bytecodes.c | 50 ++ Python/optimizer_cases.c.h | 71 +++ 8 files changed, 924 insertions(+), 589 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 5a165a5a3a2675..c1fd0cb9985adb 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1069,7 +1069,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG }, [BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, @@ -1101,7 +1101,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [CALL_KW_NON_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_LEN] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1120,7 +1120,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONTAINS_OP_SET] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, @@ -1265,7 +1265,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [STORE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1280,9 +1280,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1316,11 +1316,11 @@ _PyOpcode_macro_expansion[256] = { [BINARY_OP_INPLACE_ADD_UNICODE] = { .nuops = 3, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_INPLACE_ADD_UNICODE, OPARG_SIMPLE, 5 } } }, [BINARY_OP_MULTIPLY_FLOAT] = { .nuops = 3, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_MULTIPLY_FLOAT, OPARG_SIMPLE, 5 } } }, [BINARY_OP_MULTIPLY_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_MULTIPLY_INT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_DICT] = { .nuops = 1, .uops = { { _BINARY_OP_SUBSCR_DICT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_DICT] = { .nuops = 2, .uops = { { _GUARD_NOS_DICT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_DICT, OPARG_SIMPLE, 5 } } }, [BINARY_OP_SUBSCR_GETITEM] = { .nuops = 4, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 5 }, { _BINARY_OP_SUBSCR_CHECK_FUNC, OPARG_SIMPLE, 5 }, { _BINARY_OP_SUBSCR_INIT_CALL, OPARG_SIMPLE, 5 }, { _PUSH_FRAME, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_LIST_INT] = { .nuops = 2, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_LIST_INT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_LIST_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_LIST_INT, OPARG_SIMPLE, 5 } } }, [BINARY_OP_SUBSCR_STR_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_STR_INT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_TUPLE_INT] = { .nuops = 2, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_TUPLE_INT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_TUPLE_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_TUPLE, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_TUPLE_INT, OPARG_SIMPLE, 5 } } }, [BINARY_OP_SUBTRACT_FLOAT] = { .nuops = 3, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_FLOAT, OPARG_SIMPLE, 5 } } }, [BINARY_OP_SUBTRACT_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_INT, OPARG_SIMPLE, 5 } } }, [BINARY_SLICE] = { .nuops = 1, .uops = { { _BINARY_SLICE, OPARG_SIMPLE, 0 } } }, @@ -1344,7 +1344,7 @@ _PyOpcode_macro_expansion[256] = { [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, OPARG_SIMPLE, 3 }, { _CALL_KW_NON_PY, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, [CALL_KW_PY] = { .nuops = 5, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_LEN] = { .nuops = 1, .uops = { { _CALL_LEN, OPARG_SIMPLE, 3 } } }, - [CALL_LIST_APPEND] = { .nuops = 1, .uops = { { _CALL_LIST_APPEND, OPARG_SIMPLE, 3 } } }, + [CALL_LIST_APPEND] = { .nuops = 2, .uops = { { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _CALL_LIST_APPEND, OPARG_SIMPLE, 3 } } }, [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, @@ -1362,8 +1362,8 @@ _PyOpcode_macro_expansion[256] = { [COMPARE_OP_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _COMPARE_OP_INT, OPARG_SIMPLE, 1 } } }, [COMPARE_OP_STR] = { .nuops = 3, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _COMPARE_OP_STR, OPARG_SIMPLE, 1 } } }, [CONTAINS_OP] = { .nuops = 1, .uops = { { _CONTAINS_OP, OPARG_SIMPLE, 0 } } }, - [CONTAINS_OP_DICT] = { .nuops = 1, .uops = { { _CONTAINS_OP_DICT, OPARG_SIMPLE, 1 } } }, - [CONTAINS_OP_SET] = { .nuops = 1, .uops = { { _CONTAINS_OP_SET, OPARG_SIMPLE, 1 } } }, + [CONTAINS_OP_DICT] = { .nuops = 2, .uops = { { _GUARD_TOS_DICT, OPARG_SIMPLE, 0 }, { _CONTAINS_OP_DICT, OPARG_SIMPLE, 1 } } }, + [CONTAINS_OP_SET] = { .nuops = 2, .uops = { { _GUARD_TOS_ANY_SET, OPARG_SIMPLE, 0 }, { _CONTAINS_OP_SET, OPARG_SIMPLE, 1 } } }, [CONVERT_VALUE] = { .nuops = 1, .uops = { { _CONVERT_VALUE, OPARG_SIMPLE, 0 } } }, [COPY] = { .nuops = 1, .uops = { { _COPY, OPARG_SIMPLE, 0 } } }, [COPY_FREE_VARS] = { .nuops = 1, .uops = { { _COPY_FREE_VARS, OPARG_SIMPLE, 0 } } }, @@ -1468,14 +1468,14 @@ _PyOpcode_macro_expansion[256] = { [STORE_NAME] = { .nuops = 1, .uops = { { _STORE_NAME, OPARG_SIMPLE, 0 } } }, [STORE_SLICE] = { .nuops = 1, .uops = { { _STORE_SLICE, OPARG_SIMPLE, 0 } } }, [STORE_SUBSCR] = { .nuops = 1, .uops = { { _STORE_SUBSCR, OPARG_SIMPLE, 0 } } }, - [STORE_SUBSCR_DICT] = { .nuops = 1, .uops = { { _STORE_SUBSCR_DICT, OPARG_SIMPLE, 1 } } }, - [STORE_SUBSCR_LIST_INT] = { .nuops = 2, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_LIST_INT, OPARG_SIMPLE, 1 } } }, + [STORE_SUBSCR_DICT] = { .nuops = 2, .uops = { { _GUARD_NOS_DICT, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_DICT, OPARG_SIMPLE, 1 } } }, + [STORE_SUBSCR_LIST_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_LIST_INT, OPARG_SIMPLE, 1 } } }, [SWAP] = { .nuops = 1, .uops = { { _SWAP, OPARG_SIMPLE, 0 } } }, [TO_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL, OPARG_SIMPLE, 2 } } }, [TO_BOOL_ALWAYS_TRUE] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _REPLACE_WITH_TRUE, OPARG_SIMPLE, 3 } } }, [TO_BOOL_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL_BOOL, OPARG_SIMPLE, 3 } } }, [TO_BOOL_INT] = { .nuops = 1, .uops = { { _TO_BOOL_INT, OPARG_SIMPLE, 3 } } }, - [TO_BOOL_LIST] = { .nuops = 1, .uops = { { _TO_BOOL_LIST, OPARG_SIMPLE, 3 } } }, + [TO_BOOL_LIST] = { .nuops = 2, .uops = { { _GUARD_TOS_LIST, OPARG_SIMPLE, 0 }, { _TO_BOOL_LIST, OPARG_SIMPLE, 3 } } }, [TO_BOOL_NONE] = { .nuops = 1, .uops = { { _TO_BOOL_NONE, OPARG_SIMPLE, 3 } } }, [TO_BOOL_STR] = { .nuops = 2, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _TO_BOOL_STR, OPARG_SIMPLE, 3 } } }, [UNARY_INVERT] = { .nuops = 1, .uops = { { _UNARY_INVERT, OPARG_SIMPLE, 0 } } }, @@ -1483,9 +1483,9 @@ _PyOpcode_macro_expansion[256] = { [UNARY_NOT] = { .nuops = 1, .uops = { { _UNARY_NOT, OPARG_SIMPLE, 0 } } }, [UNPACK_EX] = { .nuops = 1, .uops = { { _UNPACK_EX, OPARG_SIMPLE, 0 } } }, [UNPACK_SEQUENCE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE, OPARG_SIMPLE, 0 } } }, - [UNPACK_SEQUENCE_LIST] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE_LIST, OPARG_SIMPLE, 1 } } }, - [UNPACK_SEQUENCE_TUPLE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE_TUPLE, OPARG_SIMPLE, 1 } } }, - [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE_TWO_TUPLE, OPARG_SIMPLE, 1 } } }, + [UNPACK_SEQUENCE_LIST] = { .nuops = 2, .uops = { { _GUARD_TOS_LIST, OPARG_SIMPLE, 0 }, { _UNPACK_SEQUENCE_LIST, OPARG_SIMPLE, 1 } } }, + [UNPACK_SEQUENCE_TUPLE] = { .nuops = 2, .uops = { { _GUARD_TOS_TUPLE, OPARG_SIMPLE, 0 }, { _UNPACK_SEQUENCE_TUPLE, OPARG_SIMPLE, 1 } } }, + [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 2, .uops = { { _GUARD_TOS_TUPLE, OPARG_SIMPLE, 0 }, { _UNPACK_SEQUENCE_TWO_TUPLE, OPARG_SIMPLE, 1 } } }, [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { _WITH_EXCEPT_START, OPARG_SIMPLE, 0 } } }, [YIELD_VALUE] = { .nuops = 1, .uops = { { _YIELD_VALUE, OPARG_SIMPLE, 0 } } }, }; diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index daac41b3d2879d..5a3bb242ec8e22 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -20,131 +20,138 @@ extern "C" { #define _BINARY_OP_MULTIPLY_FLOAT 308 #define _BINARY_OP_MULTIPLY_INT 309 #define _BINARY_OP_SUBSCR_CHECK_FUNC 310 -#define _BINARY_OP_SUBSCR_DICT BINARY_OP_SUBSCR_DICT -#define _BINARY_OP_SUBSCR_INIT_CALL 311 -#define _BINARY_OP_SUBSCR_LIST_INT 312 -#define _BINARY_OP_SUBSCR_STR_INT 313 -#define _BINARY_OP_SUBSCR_TUPLE_INT 314 -#define _BINARY_OP_SUBTRACT_FLOAT 315 -#define _BINARY_OP_SUBTRACT_INT 316 -#define _BINARY_SLICE 317 +#define _BINARY_OP_SUBSCR_DICT 311 +#define _BINARY_OP_SUBSCR_INIT_CALL 312 +#define _BINARY_OP_SUBSCR_LIST_INT 313 +#define _BINARY_OP_SUBSCR_STR_INT 314 +#define _BINARY_OP_SUBSCR_TUPLE_INT 315 +#define _BINARY_OP_SUBTRACT_FLOAT 316 +#define _BINARY_OP_SUBTRACT_INT 317 +#define _BINARY_SLICE 318 #define _BUILD_LIST BUILD_LIST #define _BUILD_MAP BUILD_MAP #define _BUILD_SET BUILD_SET #define _BUILD_SLICE BUILD_SLICE #define _BUILD_STRING BUILD_STRING #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL_BUILTIN_CLASS 318 -#define _CALL_BUILTIN_FAST 319 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 320 -#define _CALL_BUILTIN_O 321 +#define _CALL_BUILTIN_CLASS 319 +#define _CALL_BUILTIN_FAST 320 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 321 +#define _CALL_BUILTIN_O 322 #define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 #define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 #define _CALL_ISINSTANCE CALL_ISINSTANCE -#define _CALL_KW_NON_PY 322 +#define _CALL_KW_NON_PY 323 #define _CALL_LEN CALL_LEN -#define _CALL_LIST_APPEND CALL_LIST_APPEND -#define _CALL_METHOD_DESCRIPTOR_FAST 323 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 324 -#define _CALL_METHOD_DESCRIPTOR_NOARGS 325 -#define _CALL_METHOD_DESCRIPTOR_O 326 -#define _CALL_NON_PY_GENERAL 327 -#define _CALL_STR_1 328 -#define _CALL_TUPLE_1 329 +#define _CALL_LIST_APPEND 324 +#define _CALL_METHOD_DESCRIPTOR_FAST 325 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 326 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 327 +#define _CALL_METHOD_DESCRIPTOR_O 328 +#define _CALL_NON_PY_GENERAL 329 +#define _CALL_STR_1 330 +#define _CALL_TUPLE_1 331 #define _CALL_TYPE_1 CALL_TYPE_1 -#define _CHECK_AND_ALLOCATE_OBJECT 330 -#define _CHECK_ATTR_CLASS 331 -#define _CHECK_ATTR_METHOD_LAZY_DICT 332 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 333 +#define _CHECK_AND_ALLOCATE_OBJECT 332 +#define _CHECK_ATTR_CLASS 333 +#define _CHECK_ATTR_METHOD_LAZY_DICT 334 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 335 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 334 -#define _CHECK_FUNCTION_EXACT_ARGS 335 -#define _CHECK_FUNCTION_VERSION 336 -#define _CHECK_FUNCTION_VERSION_INLINE 337 -#define _CHECK_FUNCTION_VERSION_KW 338 -#define _CHECK_IS_NOT_PY_CALLABLE 339 -#define _CHECK_IS_NOT_PY_CALLABLE_KW 340 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 341 -#define _CHECK_METHOD_VERSION 342 -#define _CHECK_METHOD_VERSION_KW 343 -#define _CHECK_PEP_523 344 -#define _CHECK_PERIODIC 345 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 346 -#define _CHECK_STACK_SPACE 347 -#define _CHECK_STACK_SPACE_OPERAND 348 -#define _CHECK_VALIDITY 349 -#define _CHECK_VALIDITY_AND_SET_IP 350 -#define _COMPARE_OP 351 -#define _COMPARE_OP_FLOAT 352 -#define _COMPARE_OP_INT 353 -#define _COMPARE_OP_STR 354 -#define _CONTAINS_OP 355 -#define _CONTAINS_OP_DICT CONTAINS_OP_DICT -#define _CONTAINS_OP_SET CONTAINS_OP_SET +#define _CHECK_FUNCTION 336 +#define _CHECK_FUNCTION_EXACT_ARGS 337 +#define _CHECK_FUNCTION_VERSION 338 +#define _CHECK_FUNCTION_VERSION_INLINE 339 +#define _CHECK_FUNCTION_VERSION_KW 340 +#define _CHECK_IS_NOT_PY_CALLABLE 341 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 342 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 343 +#define _CHECK_METHOD_VERSION 344 +#define _CHECK_METHOD_VERSION_KW 345 +#define _CHECK_PEP_523 346 +#define _CHECK_PERIODIC 347 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 348 +#define _CHECK_STACK_SPACE 349 +#define _CHECK_STACK_SPACE_OPERAND 350 +#define _CHECK_VALIDITY 351 +#define _CHECK_VALIDITY_AND_SET_IP 352 +#define _COMPARE_OP 353 +#define _COMPARE_OP_FLOAT 354 +#define _COMPARE_OP_INT 355 +#define _COMPARE_OP_STR 356 +#define _CONTAINS_OP 357 +#define _CONTAINS_OP_DICT 358 +#define _CONTAINS_OP_SET 359 #define _CONVERT_VALUE CONVERT_VALUE #define _COPY COPY #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 356 +#define _CREATE_INIT_FRAME 360 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 357 +#define _DEOPT 361 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 358 -#define _DO_CALL_FUNCTION_EX 359 -#define _DO_CALL_KW 360 +#define _DO_CALL 362 +#define _DO_CALL_FUNCTION_EX 363 +#define _DO_CALL_KW 364 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 361 +#define _ERROR_POP_N 365 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 362 -#define _EXPAND_METHOD_KW 363 -#define _FATAL_ERROR 364 +#define _EXPAND_METHOD 366 +#define _EXPAND_METHOD_KW 367 +#define _FATAL_ERROR 368 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 365 -#define _FOR_ITER_GEN_FRAME 366 -#define _FOR_ITER_TIER_TWO 367 +#define _FOR_ITER 369 +#define _FOR_ITER_GEN_FRAME 370 +#define _FOR_ITER_TIER_TWO 371 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 368 -#define _GUARD_DORV_NO_DICT 369 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 370 -#define _GUARD_GLOBALS_VERSION 371 -#define _GUARD_IS_FALSE_POP 372 -#define _GUARD_IS_NONE_POP 373 -#define _GUARD_IS_NOT_NONE_POP 374 -#define _GUARD_IS_TRUE_POP 375 -#define _GUARD_KEYS_VERSION 376 -#define _GUARD_NOS_FLOAT 377 -#define _GUARD_NOS_INT 378 -#define _GUARD_NOS_UNICODE 379 -#define _GUARD_NOT_EXHAUSTED_LIST 380 -#define _GUARD_NOT_EXHAUSTED_RANGE 381 -#define _GUARD_NOT_EXHAUSTED_TUPLE 382 -#define _GUARD_TOS_FLOAT 383 -#define _GUARD_TOS_INT 384 -#define _GUARD_TOS_UNICODE 385 -#define _GUARD_TYPE_VERSION 386 -#define _GUARD_TYPE_VERSION_AND_LOCK 387 +#define _GUARD_BINARY_OP_EXTEND 372 +#define _GUARD_DORV_NO_DICT 373 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 374 +#define _GUARD_GLOBALS_VERSION 375 +#define _GUARD_IS_FALSE_POP 376 +#define _GUARD_IS_NONE_POP 377 +#define _GUARD_IS_NOT_NONE_POP 378 +#define _GUARD_IS_TRUE_POP 379 +#define _GUARD_KEYS_VERSION 380 +#define _GUARD_NOS_DICT 381 +#define _GUARD_NOS_FLOAT 382 +#define _GUARD_NOS_INT 383 +#define _GUARD_NOS_LIST 384 +#define _GUARD_NOS_TUPLE 385 +#define _GUARD_NOS_UNICODE 386 +#define _GUARD_NOT_EXHAUSTED_LIST 387 +#define _GUARD_NOT_EXHAUSTED_RANGE 388 +#define _GUARD_NOT_EXHAUSTED_TUPLE 389 +#define _GUARD_TOS_ANY_SET 390 +#define _GUARD_TOS_DICT 391 +#define _GUARD_TOS_FLOAT 392 +#define _GUARD_TOS_INT 393 +#define _GUARD_TOS_LIST 394 +#define _GUARD_TOS_TUPLE 395 +#define _GUARD_TOS_UNICODE 396 +#define _GUARD_TYPE_VERSION 397 +#define _GUARD_TYPE_VERSION_AND_LOCK 398 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 388 -#define _INIT_CALL_PY_EXACT_ARGS 389 -#define _INIT_CALL_PY_EXACT_ARGS_0 390 -#define _INIT_CALL_PY_EXACT_ARGS_1 391 -#define _INIT_CALL_PY_EXACT_ARGS_2 392 -#define _INIT_CALL_PY_EXACT_ARGS_3 393 -#define _INIT_CALL_PY_EXACT_ARGS_4 394 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 399 +#define _INIT_CALL_PY_EXACT_ARGS 400 +#define _INIT_CALL_PY_EXACT_ARGS_0 401 +#define _INIT_CALL_PY_EXACT_ARGS_1 402 +#define _INIT_CALL_PY_EXACT_ARGS_2 403 +#define _INIT_CALL_PY_EXACT_ARGS_3 404 +#define _INIT_CALL_PY_EXACT_ARGS_4 405 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -154,163 +161,163 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 395 +#define _IS_NONE 406 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 396 -#define _ITER_CHECK_RANGE 397 -#define _ITER_CHECK_TUPLE 398 -#define _ITER_JUMP_LIST 399 -#define _ITER_JUMP_RANGE 400 -#define _ITER_JUMP_TUPLE 401 -#define _ITER_NEXT_LIST 402 -#define _ITER_NEXT_LIST_TIER_TWO 403 -#define _ITER_NEXT_RANGE 404 -#define _ITER_NEXT_TUPLE 405 -#define _JUMP_TO_TOP 406 +#define _ITER_CHECK_LIST 407 +#define _ITER_CHECK_RANGE 408 +#define _ITER_CHECK_TUPLE 409 +#define _ITER_JUMP_LIST 410 +#define _ITER_JUMP_RANGE 411 +#define _ITER_JUMP_TUPLE 412 +#define _ITER_NEXT_LIST 413 +#define _ITER_NEXT_LIST_TIER_TWO 414 +#define _ITER_NEXT_RANGE 415 +#define _ITER_NEXT_TUPLE 416 +#define _JUMP_TO_TOP 417 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 407 -#define _LOAD_ATTR_CLASS 408 +#define _LOAD_ATTR 418 +#define _LOAD_ATTR_CLASS 419 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 409 -#define _LOAD_ATTR_METHOD_LAZY_DICT 410 -#define _LOAD_ATTR_METHOD_NO_DICT 411 -#define _LOAD_ATTR_METHOD_WITH_VALUES 412 -#define _LOAD_ATTR_MODULE 413 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 414 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 415 -#define _LOAD_ATTR_PROPERTY_FRAME 416 -#define _LOAD_ATTR_SLOT 417 -#define _LOAD_ATTR_WITH_HINT 418 +#define _LOAD_ATTR_INSTANCE_VALUE 420 +#define _LOAD_ATTR_METHOD_LAZY_DICT 421 +#define _LOAD_ATTR_METHOD_NO_DICT 422 +#define _LOAD_ATTR_METHOD_WITH_VALUES 423 +#define _LOAD_ATTR_MODULE 424 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 425 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 426 +#define _LOAD_ATTR_PROPERTY_FRAME 427 +#define _LOAD_ATTR_SLOT 428 +#define _LOAD_ATTR_WITH_HINT 429 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 419 +#define _LOAD_BYTECODE 430 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST #define _LOAD_CONST_IMMORTAL LOAD_CONST_IMMORTAL -#define _LOAD_CONST_INLINE 420 -#define _LOAD_CONST_INLINE_BORROW 421 +#define _LOAD_CONST_INLINE 431 +#define _LOAD_CONST_INLINE_BORROW 432 #define _LOAD_CONST_MORTAL LOAD_CONST_MORTAL #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 422 -#define _LOAD_FAST_0 423 -#define _LOAD_FAST_1 424 -#define _LOAD_FAST_2 425 -#define _LOAD_FAST_3 426 -#define _LOAD_FAST_4 427 -#define _LOAD_FAST_5 428 -#define _LOAD_FAST_6 429 -#define _LOAD_FAST_7 430 +#define _LOAD_FAST 433 +#define _LOAD_FAST_0 434 +#define _LOAD_FAST_1 435 +#define _LOAD_FAST_2 436 +#define _LOAD_FAST_3 437 +#define _LOAD_FAST_4 438 +#define _LOAD_FAST_5 439 +#define _LOAD_FAST_6 440 +#define _LOAD_FAST_7 441 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 431 -#define _LOAD_FAST_BORROW_0 432 -#define _LOAD_FAST_BORROW_1 433 -#define _LOAD_FAST_BORROW_2 434 -#define _LOAD_FAST_BORROW_3 435 -#define _LOAD_FAST_BORROW_4 436 -#define _LOAD_FAST_BORROW_5 437 -#define _LOAD_FAST_BORROW_6 438 -#define _LOAD_FAST_BORROW_7 439 +#define _LOAD_FAST_BORROW 442 +#define _LOAD_FAST_BORROW_0 443 +#define _LOAD_FAST_BORROW_1 444 +#define _LOAD_FAST_BORROW_2 445 +#define _LOAD_FAST_BORROW_3 446 +#define _LOAD_FAST_BORROW_4 447 +#define _LOAD_FAST_BORROW_5 448 +#define _LOAD_FAST_BORROW_6 449 +#define _LOAD_FAST_BORROW_7 450 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 440 -#define _LOAD_GLOBAL_BUILTINS 441 -#define _LOAD_GLOBAL_MODULE 442 +#define _LOAD_GLOBAL 451 +#define _LOAD_GLOBAL_BUILTINS 452 +#define _LOAD_GLOBAL_MODULE 453 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 443 -#define _LOAD_SMALL_INT_0 444 -#define _LOAD_SMALL_INT_1 445 -#define _LOAD_SMALL_INT_2 446 -#define _LOAD_SMALL_INT_3 447 +#define _LOAD_SMALL_INT 454 +#define _LOAD_SMALL_INT_0 455 +#define _LOAD_SMALL_INT_1 456 +#define _LOAD_SMALL_INT_2 457 +#define _LOAD_SMALL_INT_3 458 #define _LOAD_SPECIAL LOAD_SPECIAL #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 448 +#define _MAKE_CALLARGS_A_TUPLE 459 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 449 +#define _MAKE_WARM 460 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 450 -#define _MAYBE_EXPAND_METHOD_KW 451 -#define _MONITOR_CALL 452 -#define _MONITOR_CALL_KW 453 -#define _MONITOR_JUMP_BACKWARD 454 -#define _MONITOR_RESUME 455 +#define _MAYBE_EXPAND_METHOD 461 +#define _MAYBE_EXPAND_METHOD_KW 462 +#define _MONITOR_CALL 463 +#define _MONITOR_CALL_KW 464 +#define _MONITOR_JUMP_BACKWARD 465 +#define _MONITOR_RESUME 466 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 456 -#define _POP_JUMP_IF_TRUE 457 +#define _POP_JUMP_IF_FALSE 467 +#define _POP_JUMP_IF_TRUE 468 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE 458 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 459 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 460 +#define _POP_TOP_LOAD_CONST_INLINE 469 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 470 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 471 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 461 +#define _PUSH_FRAME 472 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 462 -#define _PY_FRAME_GENERAL 463 -#define _PY_FRAME_KW 464 -#define _QUICKEN_RESUME 465 -#define _REPLACE_WITH_TRUE 466 +#define _PUSH_NULL_CONDITIONAL 473 +#define _PY_FRAME_GENERAL 474 +#define _PY_FRAME_KW 475 +#define _QUICKEN_RESUME 476 +#define _REPLACE_WITH_TRUE 477 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 467 -#define _SEND 468 -#define _SEND_GEN_FRAME 469 +#define _SAVE_RETURN_OFFSET 478 +#define _SEND 479 +#define _SEND_GEN_FRAME 480 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 470 -#define _STORE_ATTR 471 -#define _STORE_ATTR_INSTANCE_VALUE 472 -#define _STORE_ATTR_SLOT 473 -#define _STORE_ATTR_WITH_HINT 474 +#define _START_EXECUTOR 481 +#define _STORE_ATTR 482 +#define _STORE_ATTR_INSTANCE_VALUE 483 +#define _STORE_ATTR_SLOT 484 +#define _STORE_ATTR_WITH_HINT 485 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 475 -#define _STORE_FAST_0 476 -#define _STORE_FAST_1 477 -#define _STORE_FAST_2 478 -#define _STORE_FAST_3 479 -#define _STORE_FAST_4 480 -#define _STORE_FAST_5 481 -#define _STORE_FAST_6 482 -#define _STORE_FAST_7 483 +#define _STORE_FAST 486 +#define _STORE_FAST_0 487 +#define _STORE_FAST_1 488 +#define _STORE_FAST_2 489 +#define _STORE_FAST_3 490 +#define _STORE_FAST_4 491 +#define _STORE_FAST_5 492 +#define _STORE_FAST_6 493 +#define _STORE_FAST_7 494 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 484 -#define _STORE_SUBSCR 485 -#define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT -#define _STORE_SUBSCR_LIST_INT 486 +#define _STORE_SLICE 495 +#define _STORE_SUBSCR 496 +#define _STORE_SUBSCR_DICT 497 +#define _STORE_SUBSCR_LIST_INT 498 #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 487 -#define _TO_BOOL 488 +#define _TIER2_RESUME_CHECK 499 +#define _TO_BOOL 500 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST TO_BOOL_LIST +#define _TO_BOOL_LIST 501 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 489 +#define _TO_BOOL_STR 502 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 490 -#define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST -#define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE -#define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE +#define _UNPACK_SEQUENCE 503 +#define _UNPACK_SEQUENCE_LIST 504 +#define _UNPACK_SEQUENCE_TUPLE 505 +#define _UNPACK_SEQUENCE_TWO_TUPLE 506 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 490 +#define MAX_UOP_ID 506 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 95ccf4e81afeab..2340147200148d 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -72,7 +72,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_TO_BOOL_BOOL] = HAS_EXIT_FLAG, [_TO_BOOL_INT] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, - [_TO_BOOL_LIST] = HAS_EXIT_FLAG, + [_GUARD_NOS_LIST] = HAS_EXIT_FLAG, + [_GUARD_TOS_LIST] = HAS_EXIT_FLAG, + [_TO_BOOL_LIST] = 0, [_TO_BOOL_NONE] = HAS_EXIT_FLAG, [_GUARD_NOS_UNICODE] = HAS_EXIT_FLAG, [_GUARD_TOS_UNICODE] = HAS_EXIT_FLAG, @@ -97,15 +99,19 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_STR_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_NOS_TUPLE] = HAS_EXIT_FLAG, + [_GUARD_TOS_TUPLE] = HAS_EXIT_FLAG, [_BINARY_OP_SUBSCR_TUPLE_INT] = HAS_DEOPT_FLAG, - [_BINARY_OP_SUBSCR_DICT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_NOS_DICT] = HAS_EXIT_FLAG, + [_GUARD_TOS_DICT] = HAS_EXIT_FLAG, + [_BINARY_OP_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_CHECK_FUNC] = HAS_DEOPT_FLAG, [_BINARY_OP_SUBSCR_INIT_CALL] = 0, [_LIST_APPEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_SET_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_STORE_SUBSCR_DICT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_STORE_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -177,8 +183,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_COMPARE_OP_STR] = HAS_ARG_FLAG, [_IS_OP] = HAS_ARG_FLAG, [_CONTAINS_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CONTAINS_OP_SET] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CONTAINS_OP_DICT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_TOS_ANY_SET] = HAS_DEOPT_FLAG, + [_CONTAINS_OP_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CONTAINS_OP_DICT] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_EG_MATCH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_EXC_MATCH] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_IMPORT_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -413,14 +420,21 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", [_GUARD_IS_TRUE_POP] = "_GUARD_IS_TRUE_POP", [_GUARD_KEYS_VERSION] = "_GUARD_KEYS_VERSION", + [_GUARD_NOS_DICT] = "_GUARD_NOS_DICT", [_GUARD_NOS_FLOAT] = "_GUARD_NOS_FLOAT", [_GUARD_NOS_INT] = "_GUARD_NOS_INT", + [_GUARD_NOS_LIST] = "_GUARD_NOS_LIST", + [_GUARD_NOS_TUPLE] = "_GUARD_NOS_TUPLE", [_GUARD_NOS_UNICODE] = "_GUARD_NOS_UNICODE", [_GUARD_NOT_EXHAUSTED_LIST] = "_GUARD_NOT_EXHAUSTED_LIST", [_GUARD_NOT_EXHAUSTED_RANGE] = "_GUARD_NOT_EXHAUSTED_RANGE", [_GUARD_NOT_EXHAUSTED_TUPLE] = "_GUARD_NOT_EXHAUSTED_TUPLE", + [_GUARD_TOS_ANY_SET] = "_GUARD_TOS_ANY_SET", + [_GUARD_TOS_DICT] = "_GUARD_TOS_DICT", [_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT", [_GUARD_TOS_INT] = "_GUARD_TOS_INT", + [_GUARD_TOS_LIST] = "_GUARD_TOS_LIST", + [_GUARD_TOS_TUPLE] = "_GUARD_TOS_TUPLE", [_GUARD_TOS_UNICODE] = "_GUARD_TOS_UNICODE", [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", [_GUARD_TYPE_VERSION_AND_LOCK] = "_GUARD_TYPE_VERSION_AND_LOCK", @@ -684,6 +698,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _TO_BOOL_INT: return 1; + case _GUARD_NOS_LIST: + return 0; + case _GUARD_TOS_LIST: + return 0; case _TO_BOOL_LIST: return 1; case _TO_BOOL_NONE: @@ -734,8 +752,16 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _BINARY_OP_SUBSCR_STR_INT: return 2; + case _GUARD_NOS_TUPLE: + return 0; + case _GUARD_TOS_TUPLE: + return 0; case _BINARY_OP_SUBSCR_TUPLE_INT: return 2; + case _GUARD_NOS_DICT: + return 0; + case _GUARD_TOS_DICT: + return 0; case _BINARY_OP_SUBSCR_DICT: return 2; case _BINARY_OP_SUBSCR_CHECK_FUNC: @@ -894,6 +920,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _CONTAINS_OP: return 2; + case _GUARD_TOS_ANY_SET: + return 0; case _CONTAINS_OP_SET: return 2; case _CONTAINS_OP_DICT: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c50df896cc9322..22a5c2100c7cf9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -508,9 +508,21 @@ dummy_func( } } - inst(TO_BOOL_LIST, (unused/1, unused/2, value -- res)) { + op(_GUARD_NOS_LIST, (nos, unused -- nos, unused)) { + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + EXIT_IF(!PyList_CheckExact(o)); + } + + op(_GUARD_TOS_LIST, (tos -- tos)) { + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + EXIT_IF(!PyList_CheckExact(o)); + } + + macro(TO_BOOL_LIST) = _GUARD_TOS_LIST + unused/1 + unused/2 + _TO_BOOL_LIST; + + op(_TO_BOOL_LIST, (value -- res)) { PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - EXIT_IF(!PyList_CheckExact(value_o)); + assert(PyList_CheckExact(value_o)); STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; DECREF_INPUTS(); @@ -524,8 +536,7 @@ dummy_func( res = PyStackRef_False; } - op(_GUARD_NOS_UNICODE, (nos, tos -- nos, tos)) { - (void)tos; + op(_GUARD_NOS_UNICODE, (nos, unused -- nos, unused)) { PyObject *o = PyStackRef_AsPyObjectBorrow(nos); EXIT_IF(!PyUnicode_CheckExact(o)); } @@ -587,8 +598,7 @@ dummy_func( BINARY_OP_EXTEND, }; - op(_GUARD_NOS_INT, (left, right -- left, right)) { - (void)right; + op(_GUARD_NOS_INT, (left, unused -- left, unused)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); EXIT_IF(!PyLong_CheckExact(left_o)); } @@ -650,8 +660,7 @@ dummy_func( macro(BINARY_OP_SUBTRACT_INT) = _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_SUBTRACT_INT; - op(_GUARD_NOS_FLOAT, (left, right -- left, right)) { - (void)right; + op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); EXIT_IF(!PyFloat_CheckExact(left_o)); } @@ -862,14 +871,14 @@ dummy_func( macro(STORE_SLICE) = _SPECIALIZE_STORE_SLICE + _STORE_SLICE; macro(BINARY_OP_SUBSCR_LIST_INT) = - _GUARD_TOS_INT + unused/5 + _BINARY_OP_SUBSCR_LIST_INT; + _GUARD_TOS_INT + _GUARD_NOS_LIST + unused/5 + _BINARY_OP_SUBSCR_LIST_INT; op(_BINARY_OP_SUBSCR_LIST_INT, (list_st, sub_st -- res)) { PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); assert(PyLong_CheckExact(sub)); - DEOPT_IF(!PyList_CheckExact(list)); + assert(PyList_CheckExact(list)); // Deopt unless 0 <= sub < PyList_Size(list) DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); @@ -913,15 +922,25 @@ dummy_func( res = PyStackRef_FromPyObjectImmortal(res_o); } + op(_GUARD_NOS_TUPLE, (nos, unused -- nos, unused)) { + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + EXIT_IF(!PyTuple_CheckExact(o)); + } + + op(_GUARD_TOS_TUPLE, (tos -- tos)) { + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + EXIT_IF(!PyTuple_CheckExact(o)); + } + macro(BINARY_OP_SUBSCR_TUPLE_INT) = - _GUARD_TOS_INT + unused/5 + _BINARY_OP_SUBSCR_TUPLE_INT; + _GUARD_TOS_INT + _GUARD_NOS_TUPLE + unused/5 + _BINARY_OP_SUBSCR_TUPLE_INT; op(_BINARY_OP_SUBSCR_TUPLE_INT, (tuple_st, sub_st -- res)) { PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); assert(PyLong_CheckExact(sub)); - DEOPT_IF(!PyTuple_CheckExact(tuple)); + assert(PyTuple_CheckExact(tuple)); // Deopt unless 0 <= sub < PyTuple_Size(list) DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); @@ -935,11 +954,24 @@ dummy_func( DECREF_INPUTS(); } - inst(BINARY_OP_SUBSCR_DICT, (unused/5, dict_st, sub_st -- res)) { + op(_GUARD_NOS_DICT, (nos, unused -- nos, unused)) { + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + EXIT_IF(!PyDict_CheckExact(o)); + } + + op(_GUARD_TOS_DICT, (tos -- tos)) { + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + EXIT_IF(!PyDict_CheckExact(o)); + } + + macro(BINARY_OP_SUBSCR_DICT) = + _GUARD_NOS_DICT + unused/5 + _BINARY_OP_SUBSCR_DICT; + + op(_BINARY_OP_SUBSCR_DICT, (dict_st, sub_st -- res)) { PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict)); + assert(PyDict_CheckExact(dict)); STAT_INC(BINARY_OP, hit); PyObject *res_o; int rc = PyDict_GetItemRef(dict, sub, &res_o); @@ -1021,14 +1053,14 @@ dummy_func( macro(STORE_SUBSCR) = _SPECIALIZE_STORE_SUBSCR + _STORE_SUBSCR; macro(STORE_SUBSCR_LIST_INT) = - _GUARD_TOS_INT + unused/1 + _STORE_SUBSCR_LIST_INT; + _GUARD_TOS_INT + _GUARD_NOS_LIST + unused/1 + _STORE_SUBSCR_LIST_INT; op(_STORE_SUBSCR_LIST_INT, (value, list_st, sub_st -- )) { PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); assert(PyLong_CheckExact(sub)); - DEOPT_IF(!PyList_CheckExact(list)); + assert(PyList_CheckExact(list)); // Ensure nonnegative, zero-or-one-digit ints. DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); @@ -1052,10 +1084,13 @@ dummy_func( Py_DECREF(old_value); } - inst(STORE_SUBSCR_DICT, (unused/1, value, dict_st, sub -- )) { + macro(STORE_SUBSCR_DICT) = + _GUARD_NOS_DICT + unused/1 + _STORE_SUBSCR_DICT; + + op(_STORE_SUBSCR_DICT, (value, dict_st, sub -- )) { PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - DEOPT_IF(!PyDict_CheckExact(dict)); + assert(PyDict_CheckExact(dict)); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(sub), @@ -1501,10 +1536,13 @@ dummy_func( macro(UNPACK_SEQUENCE) = _SPECIALIZE_UNPACK_SEQUENCE + _UNPACK_SEQUENCE; - inst(UNPACK_SEQUENCE_TWO_TUPLE, (unused/1, seq -- val1, val0)) { + macro(UNPACK_SEQUENCE_TWO_TUPLE) = + _GUARD_TOS_TUPLE + unused/1 + _UNPACK_SEQUENCE_TWO_TUPLE; + + op(_UNPACK_SEQUENCE_TWO_TUPLE, (seq -- val1, val0)) { assert(oparg == 2); PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o)); + assert(PyTuple_CheckExact(seq_o)); DEOPT_IF(PyTuple_GET_SIZE(seq_o) != 2); STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); @@ -1512,9 +1550,12 @@ dummy_func( PyStackRef_CLOSE(seq); } - inst(UNPACK_SEQUENCE_TUPLE, (unused/1, seq -- values[oparg])) { + macro(UNPACK_SEQUENCE_TUPLE) = + _GUARD_TOS_TUPLE + unused/1 + _UNPACK_SEQUENCE_TUPLE; + + op(_UNPACK_SEQUENCE_TUPLE, (seq -- values[oparg])) { PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyTuple_CheckExact(seq_o)); + assert(PyTuple_CheckExact(seq_o)); DEOPT_IF(PyTuple_GET_SIZE(seq_o) != oparg); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq_o); @@ -1524,9 +1565,12 @@ dummy_func( DECREF_INPUTS(); } - inst(UNPACK_SEQUENCE_LIST, (unused/1, seq -- values[oparg])) { + macro(UNPACK_SEQUENCE_LIST) = + _GUARD_TOS_LIST + unused/1 + _UNPACK_SEQUENCE_LIST; + + op(_UNPACK_SEQUENCE_LIST, (seq -- values[oparg])) { PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - DEOPT_IF(!PyList_CheckExact(seq_o)); + assert(PyList_CheckExact(seq_o)); DEOPT_IF(!LOCK_OBJECT(seq_o)); if (PyList_GET_SIZE(seq_o) != oparg) { UNLOCK_OBJECT(seq_o); @@ -2683,11 +2727,18 @@ dummy_func( macro(CONTAINS_OP) = _SPECIALIZE_CONTAINS_OP + _CONTAINS_OP; - inst(CONTAINS_OP_SET, (unused/1, left, right -- b)) { + op(_GUARD_TOS_ANY_SET, (tos -- tos)) { + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + DEOPT_IF(!PyAnySet_CheckExact(o)); + } + + macro(CONTAINS_OP_SET) = _GUARD_TOS_ANY_SET + unused/1 + _CONTAINS_OP_SET; + + op(_CONTAINS_OP_SET, (left, right -- b)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))); + assert(PyAnySet_CheckExact(right_o)); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! int res = _PySet_Contains((PySetObject *)right_o, left_o); @@ -2696,11 +2747,13 @@ dummy_func( b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } - inst(CONTAINS_OP_DICT, (unused/1, left, right -- b)) { + macro(CONTAINS_OP_DICT) = _GUARD_TOS_DICT + unused/1 + _CONTAINS_OP_DICT; + + op(_CONTAINS_OP_DICT, (left, right -- b)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - DEOPT_IF(!PyDict_CheckExact(right_o)); + assert(PyDict_CheckExact(right_o)); STAT_INC(CONTAINS_OP, hit); int res = PyDict_Contains(right_o, left_o); DECREF_INPUTS(); @@ -4225,8 +4278,11 @@ dummy_func( DECREF_INPUTS(); } + macro(CALL_LIST_APPEND) = + _GUARD_NOS_LIST + unused/1 + unused/2 + _CALL_LIST_APPEND; + // This is secretly a super-instruction - inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, arg -- )) { + op(_CALL_LIST_APPEND, (callable, self, arg -- )) { assert(oparg == 1); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); @@ -4234,7 +4290,7 @@ dummy_func( PyInterpreterState *interp = tstate->interp; DEOPT_IF(callable_o != interp->callable_cache.list_append); assert(self_o != NULL); - DEOPT_IF(!PyList_Check(self_o)); + assert(PyList_CheckExact(self_o)); DEOPT_IF(!LOCK_OBJECT(self_o)); STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 117c80c8d78dbc..461c0c19825306 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -725,15 +725,34 @@ break; } + case _GUARD_NOS_LIST: { + _PyStackRef nos; + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _GUARD_TOS_LIST: { + _PyStackRef tos; + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + case _TO_BOOL_LIST: { _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyList_CheckExact(value_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + assert(PyList_CheckExact(value_o)); STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -761,11 +780,8 @@ } case _GUARD_NOS_UNICODE: { - _PyStackRef tos; _PyStackRef nos; - tos = stack_pointer[-1]; nos = stack_pointer[-2]; - (void)tos; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyUnicode_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); @@ -849,11 +865,8 @@ } case _GUARD_NOS_INT: { - _PyStackRef right; _PyStackRef left; - right = stack_pointer[-1]; left = stack_pointer[-2]; - (void)right; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!PyLong_CheckExact(left_o)) { UOP_STAT_INC(uopcode, miss); @@ -958,11 +971,8 @@ } case _GUARD_NOS_FLOAT: { - _PyStackRef right; _PyStackRef left; - right = stack_pointer[-1]; left = stack_pointer[-2]; - (void)right; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!PyFloat_CheckExact(left_o)) { UOP_STAT_INC(uopcode, miss); @@ -1299,10 +1309,7 @@ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); assert(PyLong_CheckExact(sub)); - if (!PyList_CheckExact(list)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + assert(PyList_CheckExact(list)); // Deopt unless 0 <= sub < PyList_Size(list) if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UOP_STAT_INC(uopcode, miss); @@ -1385,6 +1392,28 @@ break; } + case _GUARD_NOS_TUPLE: { + _PyStackRef nos; + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyTuple_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _GUARD_TOS_TUPLE: { + _PyStackRef tos; + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + case _BINARY_OP_SUBSCR_TUPLE_INT: { _PyStackRef sub_st; _PyStackRef tuple_st; @@ -1394,10 +1423,7 @@ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); assert(PyLong_CheckExact(sub)); - if (!PyTuple_CheckExact(tuple)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + assert(PyTuple_CheckExact(tuple)); // Deopt unless 0 <= sub < PyTuple_Size(list) if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UOP_STAT_INC(uopcode, miss); @@ -1424,6 +1450,28 @@ break; } + case _GUARD_NOS_DICT: { + _PyStackRef nos; + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _GUARD_TOS_DICT: { + _PyStackRef tos; + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + case _BINARY_OP_SUBSCR_DICT: { _PyStackRef sub_st; _PyStackRef dict_st; @@ -1432,10 +1480,7 @@ dict_st = stack_pointer[-2]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - if (!PyDict_CheckExact(dict)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + assert(PyDict_CheckExact(dict)); STAT_INC(BINARY_OP, hit); PyObject *res_o; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1601,10 +1646,7 @@ PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); assert(PyLong_CheckExact(sub)); - if (!PyList_CheckExact(list)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + assert(PyList_CheckExact(list)); // Ensure nonnegative, zero-or-one-digit ints. if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UOP_STAT_INC(uopcode, miss); @@ -1647,10 +1689,7 @@ dict_st = stack_pointer[-2]; value = stack_pointer[-3]; PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - if (!PyDict_CheckExact(dict)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + assert(PyDict_CheckExact(dict)); STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, @@ -2090,10 +2129,7 @@ seq = stack_pointer[-1]; assert(oparg == 2); PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - if (!PyTuple_CheckExact(seq_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + assert(PyTuple_CheckExact(seq_o)); if (PyTuple_GET_SIZE(seq_o) != 2) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -2119,10 +2155,7 @@ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - if (!PyTuple_CheckExact(seq_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + assert(PyTuple_CheckExact(seq_o)); if (PyTuple_GET_SIZE(seq_o) != oparg) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -2147,10 +2180,7 @@ seq = stack_pointer[-1]; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - if (!PyList_CheckExact(seq_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + assert(PyList_CheckExact(seq_o)); if (!LOCK_OBJECT(seq_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -3753,6 +3783,17 @@ break; } + case _GUARD_TOS_ANY_SET: { + _PyStackRef tos; + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnySet_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + case _CONTAINS_OP_SET: { _PyStackRef right; _PyStackRef left; @@ -3762,10 +3803,7 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + assert(PyAnySet_CheckExact(right_o)); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3800,10 +3838,7 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyDict_CheckExact(right_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + assert(PyDict_CheckExact(right_o)); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); @@ -5835,10 +5870,7 @@ JUMP_TO_JUMP_TARGET(); } assert(self_o != NULL); - if (!PyList_Check(self_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + assert(PyList_CheckExact(self_o)); if (!LOCK_OBJECT(self_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 41ea054d3f5eea..09361233bef537 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -108,9 +108,7 @@ } // _GUARD_NOS_FLOAT { - right = value; left = stack_pointer[-2]; - (void)right; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); @@ -121,6 +119,7 @@ /* Skip 5 cache entries */ // _BINARY_OP_ADD_FLOAT { + right = value; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); @@ -167,9 +166,7 @@ } // _GUARD_NOS_INT { - right = value; left = stack_pointer[-2]; - (void)right; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!PyLong_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); @@ -180,6 +177,7 @@ /* Skip 5 cache entries */ // _BINARY_OP_ADD_INT { + right = value; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); @@ -214,7 +212,6 @@ static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; - _PyStackRef tos; _PyStackRef left; _PyStackRef right; _PyStackRef res; @@ -230,9 +227,7 @@ } // _GUARD_NOS_UNICODE { - tos = value; nos = stack_pointer[-2]; - (void)tos; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); @@ -243,7 +238,7 @@ /* Skip 5 cache entries */ // _BINARY_OP_ADD_UNICODE { - right = tos; + right = value; left = nos; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); @@ -341,7 +336,6 @@ static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; - _PyStackRef tos; _PyStackRef left; _PyStackRef right; // _GUARD_TOS_UNICODE @@ -356,9 +350,7 @@ } // _GUARD_NOS_UNICODE { - tos = value; nos = stack_pointer[-2]; - (void)tos; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); @@ -369,7 +361,7 @@ /* Skip 5 cache entries */ // _BINARY_OP_INPLACE_ADD_UNICODE { - right = tos; + right = value; left = nos; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); assert(PyUnicode_CheckExact(left_o)); @@ -453,9 +445,7 @@ } // _GUARD_NOS_FLOAT { - right = value; left = stack_pointer[-2]; - (void)right; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); @@ -466,6 +456,7 @@ /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_FLOAT { + right = value; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); @@ -512,9 +503,7 @@ } // _GUARD_NOS_INT { - right = value; left = stack_pointer[-2]; - (void)right; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!PyLong_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); @@ -525,6 +514,7 @@ /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_INT { + right = value; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); @@ -557,46 +547,56 @@ next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef nos; _PyStackRef dict_st; _PyStackRef sub_st; _PyStackRef res; - /* Skip 5 cache entries */ - sub_st = stack_pointer[-1]; - dict_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + // _GUARD_NOS_DICT + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } } - STAT_INC(BINARY_OP, hit); - PyObject *res_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int rc = PyDict_GetItemRef(dict, sub, &res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (rc == 0) { + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_DICT + { + sub_st = stack_pointer[-1]; + dict_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetKeyError(sub); + int rc = PyDict_GetItemRef(dict, sub, &res_o); stack_pointer = _PyFrame_GetStackPointer(frame); + if (rc == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetKeyError(sub); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + tmp = dict_st; + dict_st = PyStackRef_NULL; + stack_pointer[-2] = dict_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (rc <= 0) { + JUMP_TO_LABEL(error); + } + // not found or error + res = PyStackRef_FromPyObjectSteal(res_o); } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = sub_st; - sub_st = PyStackRef_NULL; - stack_pointer[-1] = sub_st; - PyStackRef_CLOSE(tmp); - tmp = dict_st; - dict_st = PyStackRef_NULL; - stack_pointer[-2] = dict_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (rc <= 0) { - JUMP_TO_LABEL(error); - } - // not found or error - res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -700,6 +700,7 @@ INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; + _PyStackRef nos; _PyStackRef list_st; _PyStackRef sub_st; _PyStackRef res; @@ -713,19 +714,25 @@ JUMP_TO_PREDICTED(BINARY_OP); } } + // _GUARD_NOS_LIST + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } /* Skip 5 cache entries */ // _BINARY_OP_SUBSCR_LIST_INT { sub_st = value; - list_st = stack_pointer[-2]; + list_st = nos; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); assert(PyLong_CheckExact(sub)); - if (!PyList_CheckExact(list)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); - } + assert(PyList_CheckExact(list)); // Deopt unless 0 <= sub < PyList_Size(list) if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(BINARY_OP); @@ -785,7 +792,6 @@ static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; - _PyStackRef tos; _PyStackRef str_st; _PyStackRef sub_st; _PyStackRef res; @@ -801,9 +807,7 @@ } // _GUARD_NOS_UNICODE { - tos = value; nos = stack_pointer[-2]; - (void)tos; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); @@ -814,7 +818,7 @@ /* Skip 5 cache entries */ // _BINARY_OP_SUBSCR_STR_INT { - sub_st = tos; + sub_st = value; str_st = nos; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); @@ -866,6 +870,7 @@ INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; + _PyStackRef nos; _PyStackRef tuple_st; _PyStackRef sub_st; _PyStackRef res; @@ -879,19 +884,25 @@ JUMP_TO_PREDICTED(BINARY_OP); } } + // _GUARD_NOS_TUPLE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyTuple_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } /* Skip 5 cache entries */ // _BINARY_OP_SUBSCR_TUPLE_INT { sub_st = value; - tuple_st = stack_pointer[-2]; + tuple_st = nos; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); assert(PyLong_CheckExact(sub)); - if (!PyTuple_CheckExact(tuple)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); - } + assert(PyTuple_CheckExact(tuple)); // Deopt unless 0 <= sub < PyTuple_Size(list) if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(BINARY_OP); @@ -948,9 +959,7 @@ } // _GUARD_NOS_FLOAT { - right = value; left = stack_pointer[-2]; - (void)right; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); @@ -961,6 +970,7 @@ /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_FLOAT { + right = value; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); @@ -1007,9 +1017,7 @@ } // _GUARD_NOS_INT { - right = value; left = stack_pointer[-2]; - (void)right; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!PyLong_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); @@ -1020,6 +1028,7 @@ /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_INT { + right = value; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); @@ -3325,56 +3334,66 @@ next_instr += 4; INSTRUCTION_STATS(CALL_LIST_APPEND); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef nos; _PyStackRef callable; _PyStackRef self; _PyStackRef arg; + // _GUARD_NOS_LIST + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } /* Skip 1 cache entry */ /* Skip 2 cache entries */ - arg = stack_pointer[-1]; - self = stack_pointer[-2]; - callable = stack_pointer[-3]; - assert(oparg == 1); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.list_append) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - assert(self_o != NULL); - if (!PyList_Check(self_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - if (!LOCK_OBJECT(self_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - STAT_INC(CALL, hit); - int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); - UNLOCK_OBJECT(self_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - JUMP_TO_LABEL(error); + // _CALL_LIST_APPEND + { + arg = stack_pointer[-1]; + self = nos; + callable = stack_pointer[-3]; + assert(oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.list_append) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + assert(self_o != NULL); + assert(PyList_CheckExact(self_o)); + if (!LOCK_OBJECT(self_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + #if TIER_ONE + // Skip the following POP_TOP. This is done here in tier one, and + // during trace projection in tier two: + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); + #endif } - #if TIER_ONE - // Skip the following POP_TOP. This is done here in tier one, and - // during trace projection in tier two: - assert(next_instr->op.code == POP_TOP); - SKIP_OVER(1); - #endif DISPATCH(); } @@ -4625,9 +4644,7 @@ } // _GUARD_NOS_FLOAT { - right = value; left = stack_pointer[-2]; - (void)right; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(COMPARE_OP); @@ -4638,6 +4655,7 @@ /* Skip 1 cache entry */ // _COMPARE_OP_FLOAT { + right = value; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(COMPARE_OP, hit); @@ -4683,9 +4701,7 @@ } // _GUARD_NOS_INT { - right = value; left = stack_pointer[-2]; - (void)right; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!PyLong_CheckExact(left_o)) { UPDATE_MISS_STATS(COMPARE_OP); @@ -4696,6 +4712,7 @@ /* Skip 1 cache entry */ // _COMPARE_OP_INT { + right = value; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); if (!_PyLong_IsCompact((PyLongObject *)left_o)) { @@ -4739,7 +4756,6 @@ static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; - _PyStackRef tos; _PyStackRef left; _PyStackRef right; _PyStackRef res; @@ -4755,9 +4771,7 @@ } // _GUARD_NOS_UNICODE { - tos = value; nos = stack_pointer[-2]; - (void)tos; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(COMPARE_OP); @@ -4768,7 +4782,7 @@ /* Skip 1 cache entry */ // _COMPARE_OP_STR { - right = tos; + right = value; left = nos; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); @@ -4860,37 +4874,47 @@ next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_DICT); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef tos; _PyStackRef left; _PyStackRef right; _PyStackRef b; - /* Skip 1 cache entry */ - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!PyDict_CheckExact(right_o)) { - UPDATE_MISS_STATS(CONTAINS_OP); - assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); - JUMP_TO_PREDICTED(CONTAINS_OP); + // _GUARD_TOS_DICT + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyDict_CheckExact(o)) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + JUMP_TO_PREDICTED(CONTAINS_OP); + } } - STAT_INC(CONTAINS_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyDict_Contains(right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res < 0) { - JUMP_TO_LABEL(error); + /* Skip 1 cache entry */ + // _CONTAINS_OP_DICT + { + right = tos; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyDict_CheckExact(right_o)); + STAT_INC(CONTAINS_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyDict_Contains(right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res < 0) { + JUMP_TO_LABEL(error); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -4908,38 +4932,48 @@ next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_SET); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef tos; _PyStackRef left; _PyStackRef right; _PyStackRef b; - /* Skip 1 cache entry */ - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (!(PySet_CheckExact(right_o) || PyFrozenSet_CheckExact(right_o))) { - UPDATE_MISS_STATS(CONTAINS_OP); - assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); - JUMP_TO_PREDICTED(CONTAINS_OP); + // _GUARD_TOS_ANY_SET + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnySet_CheckExact(o)) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + JUMP_TO_PREDICTED(CONTAINS_OP); + } } - STAT_INC(CONTAINS_OP, hit); - // Note: both set and frozenset use the same seq_contains method! - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PySet_Contains((PySetObject *)right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res < 0) { - JUMP_TO_LABEL(error); + /* Skip 1 cache entry */ + // _CONTAINS_OP_SET + { + right = tos; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyAnySet_CheckExact(right_o)); + STAT_INC(CONTAINS_OP, hit); + // Note: both set and frozenset use the same seq_contains method! + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PySet_Contains((PySetObject *)right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res < 0) { + JUMP_TO_LABEL(error); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -11420,32 +11454,42 @@ next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef nos; _PyStackRef value; _PyStackRef dict_st; _PyStackRef sub; - /* Skip 1 cache entry */ - sub = stack_pointer[-1]; - dict_st = stack_pointer[-2]; - value = stack_pointer[-3]; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(STORE_SUBSCR); + // _GUARD_NOS_DICT + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } } - STAT_INC(STORE_SUBSCR, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(sub), - PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - JUMP_TO_LABEL(error); + /* Skip 1 cache entry */ + // _STORE_SUBSCR_DICT + { + sub = stack_pointer[-1]; + dict_st = nos; + value = stack_pointer[-3]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + STAT_INC(STORE_SUBSCR, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(sub), + PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } } DISPATCH(); } @@ -11462,6 +11506,7 @@ INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); _PyStackRef value; + _PyStackRef nos; _PyStackRef list_st; _PyStackRef sub_st; // _GUARD_TOS_INT @@ -11474,20 +11519,26 @@ JUMP_TO_PREDICTED(STORE_SUBSCR); } } + // _GUARD_NOS_LIST + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + } /* Skip 1 cache entry */ // _STORE_SUBSCR_LIST_INT { sub_st = value; - list_st = stack_pointer[-2]; + list_st = nos; value = stack_pointer[-3]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); assert(PyLong_CheckExact(sub)); - if (!PyList_CheckExact(list)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(STORE_SUBSCR); - } + assert(PyList_CheckExact(list)); // Ensure nonnegative, zero-or-one-digit ints. if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(STORE_SUBSCR); @@ -11716,25 +11767,35 @@ next_instr += 4; INSTRUCTION_STATS(TO_BOOL_LIST); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef tos; _PyStackRef value; _PyStackRef res; + // _GUARD_TOS_LIST + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + } /* Skip 1 cache entry */ /* Skip 2 cache entries */ - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyList_CheckExact(value_o)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TO_BOOL); + // _TO_BOOL_LIST + { + value = tos; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyList_CheckExact(value_o)); + STAT_INC(TO_BOOL, hit); + res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = res; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); } - STAT_INC(TO_BOOL, hit); - res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = value; - value = res; - stack_pointer[-1] = value; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -11980,41 +12041,51 @@ next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; _PyStackRef seq; _PyStackRef *values; - /* Skip 1 cache entry */ - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - if (!PyList_CheckExact(seq_o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(UNPACK_SEQUENCE); - } - if (!LOCK_OBJECT(seq_o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(UNPACK_SEQUENCE); - } - if (PyList_GET_SIZE(seq_o) != oparg) { - UNLOCK_OBJECT(seq_o); - if (true) { + // _GUARD_TOS_LIST + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } } - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_LIST + { + seq = tos; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyList_CheckExact(seq_o)); + if (!LOCK_OBJECT(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + if (PyList_GET_SIZE(seq_o) != oparg) { + UNLOCK_OBJECT(seq_o); + if (true) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyList_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + UNLOCK_OBJECT(seq_o); + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); } - UNLOCK_OBJECT(seq_o); - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -12029,32 +12100,42 @@ next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; _PyStackRef seq; _PyStackRef *values; - /* Skip 1 cache entry */ - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - if (!PyTuple_CheckExact(seq_o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(UNPACK_SEQUENCE); - } - if (PyTuple_GET_SIZE(seq_o) != oparg) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + // _GUARD_TOS_TUPLE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } } - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_TUPLE + { + seq = tos; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != oparg) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); } - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -12069,33 +12150,43 @@ next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; _PyStackRef seq; _PyStackRef val1; _PyStackRef val0; + // _GUARD_TOS_TUPLE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + } /* Skip 1 cache entry */ - seq = stack_pointer[-1]; - assert(oparg == 2); - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - if (!PyTuple_CheckExact(seq_o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(UNPACK_SEQUENCE); - } - if (PyTuple_GET_SIZE(seq_o) != 2) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(UNPACK_SEQUENCE); - } - STAT_INC(UNPACK_SEQUENCE, hit); - val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); - val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - stack_pointer[-1] = val1; - stack_pointer[0] = val0; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + // _UNPACK_SEQUENCE_TWO_TUPLE + { + seq = tos; + assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != 2) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + } stack_pointer[-1] = val0; DISPATCH(); } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index a948cccbf858ac..832bc4530ab916 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -913,6 +913,56 @@ dummy_func(void) { } } + op(_GUARD_TOS_LIST, (tos -- tos)) { + if (sym_matches_type(tos, &PyList_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(tos, &PyList_Type); + } + + op(_GUARD_NOS_LIST, (nos, unused -- nos, unused)) { + if (sym_matches_type(nos, &PyList_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(nos, &PyList_Type); + } + + op(_GUARD_TOS_TUPLE, (tos -- tos)) { + if (sym_matches_type(tos, &PyTuple_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(tos, &PyTuple_Type); + } + + op(_GUARD_NOS_TUPLE, (nos, unused -- nos, unused)) { + if (sym_matches_type(nos, &PyTuple_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(nos, &PyTuple_Type); + } + + op(_GUARD_TOS_DICT, (tos -- tos)) { + if (sym_matches_type(tos, &PyDict_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(tos, &PyDict_Type); + } + + op(_GUARD_NOS_DICT, (nos, unused -- nos, unused)) { + if (sym_matches_type(nos, &PyDict_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(nos, &PyDict_Type); + } + + op(_GUARD_TOS_ANY_SET, (tos -- tos)) { + if (sym_matches_type(tos, &PySet_Type) || + sym_matches_type(tos, &PyFrozenSet_Type)) + { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + } + // END BYTECODES // diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 6fdc81f1e5a01d..98eb078ca07900 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -193,6 +193,26 @@ break; } + case _GUARD_NOS_LIST: { + JitOptSymbol *nos; + nos = stack_pointer[-2]; + if (sym_matches_type(nos, &PyList_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(nos, &PyList_Type); + break; + } + + case _GUARD_TOS_LIST: { + JitOptSymbol *tos; + tos = stack_pointer[-1]; + if (sym_matches_type(tos, &PyList_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(tos, &PyList_Type); + break; + } + case _TO_BOOL_LIST: { JitOptSymbol *value; JitOptSymbol *res; @@ -591,6 +611,26 @@ break; } + case _GUARD_NOS_TUPLE: { + JitOptSymbol *nos; + nos = stack_pointer[-2]; + if (sym_matches_type(nos, &PyTuple_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(nos, &PyTuple_Type); + break; + } + + case _GUARD_TOS_TUPLE: { + JitOptSymbol *tos; + tos = stack_pointer[-1]; + if (sym_matches_type(tos, &PyTuple_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(tos, &PyTuple_Type); + break; + } + case _BINARY_OP_SUBSCR_TUPLE_INT: { JitOptSymbol *res; res = sym_new_not_null(ctx); @@ -600,6 +640,26 @@ break; } + case _GUARD_NOS_DICT: { + JitOptSymbol *nos; + nos = stack_pointer[-2]; + if (sym_matches_type(nos, &PyDict_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(nos, &PyDict_Type); + break; + } + + case _GUARD_TOS_DICT: { + JitOptSymbol *tos; + tos = stack_pointer[-1]; + if (sym_matches_type(tos, &PyDict_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(tos, &PyDict_Type); + break; + } + case _BINARY_OP_SUBSCR_DICT: { JitOptSymbol *res; res = sym_new_not_null(ctx); @@ -1311,6 +1371,17 @@ break; } + case _GUARD_TOS_ANY_SET: { + JitOptSymbol *tos; + tos = stack_pointer[-1]; + if (sym_matches_type(tos, &PySet_Type) || + sym_matches_type(tos, &PyFrozenSet_Type)) + { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + break; + } + case _CONTAINS_OP_SET: { JitOptSymbol *b; b = sym_new_not_null(ctx); From 99d6288dfadf40c0d6c1664a7f77ea4b6a81f56f Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 1 Apr 2025 18:05:30 -0700 Subject: [PATCH 2/5] Remove old sym_set_type call --- Python/optimizer_bytecodes.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 832bc4530ab916..90d88c486278ac 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -388,7 +388,6 @@ dummy_func(void) { op(_TO_BOOL_LIST, (value -- res)) { if (!optimize_to_bool(this_instr, ctx, value, &res)) { - sym_set_type(value, &PyList_Type); res = sym_new_type(ctx, &PyBool_Type); } } From a471ee34479d4df3899d312533973284b810a0f7 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 8 Apr 2025 09:20:38 -0700 Subject: [PATCH 3/5] blurb add --- .../2025-04-08-09-20-18.gh-issue-131798.Xp1mvN.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-04-08-09-20-18.gh-issue-131798.Xp1mvN.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-08-09-20-18.gh-issue-131798.Xp1mvN.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-08-09-20-18.gh-issue-131798.Xp1mvN.rst new file mode 100644 index 00000000000000..91b7a6619721cd --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-08-09-20-18.gh-issue-131798.Xp1mvN.rst @@ -0,0 +1,2 @@ +Allow the JIT compiler to remove some type checks for operations on lists, +tuples, dictionaries, and sets. From 0efca44d6c2222f14446f4a04f21ed8536779d29 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 8 Apr 2025 10:24:59 -0700 Subject: [PATCH 4/5] Add regression tests --- Lib/test/test_capi/test_opt.py | 74 ++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 9ac5d1ecf7eb1e..11718e283e5fc4 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1620,6 +1620,80 @@ def f(n): self.assertIn("_TO_BOOL_STR", uops) self.assertNotIn("_GUARD_TOS_UNICODE", uops) + def test_remove_guard_for_known_type_dict(self): + def f(n): + x = 0 + for _ in range(n): + d = {} + d["Spam"] = 1 # Guarded... + x += d["Spam"] # ...unguarded! + return x + + res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertEqual(uops.count("_GUARD_NOS_DICT"), 1) + self.assertEqual(uops.count("_STORE_SUBSCR_DICT"), 1) + self.assertEqual(uops.count("_BINARY_OP_SUBSCR_DICT"), 1) + + def test_remove_guard_for_known_type_list(self): + def f(n): + x = 0 + for _ in range(n): + l = [0] + l[0] = 1 # Guarded... + [a] = l # ...unguarded! + b = l[0] # ...unguarded! + if l: # ...unguarded! + x += a + b + return x + + res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(res, 2 * TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertEqual(uops.count("_GUARD_NOS_LIST"), 1) + self.assertEqual(uops.count("_STORE_SUBSCR_LIST_INT"), 1) + self.assertEqual(uops.count("_GUARD_TOS_LIST"), 0) + self.assertEqual(uops.count("_UNPACK_SEQUENCE_LIST"), 1) + self.assertEqual(uops.count("_BINARY_OP_SUBSCR_LIST_INT"), 1) + self.assertEqual(uops.count("_TO_BOOL_LIST"), 1) + + def test_remove_guard_for_known_type_set(self): + def f(n): + x = 0 + for _ in range(n): + x += "Spam" in {"Spam"} # Unguarded! + return x + + res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_TOS_ANY_SET", uops) + self.assertIn("_CONTAINS_OP_SET", uops) + + def test_remove_guard_for_known_type_tuple(self): + def f(n): + x = 0 + for _ in range(n): + t = (1, 2, (3, (4,))) + t_0, t_1, (t_2_0, t_2_1) = t # Unguarded! + t_2_1_0 = t_2_1[0] # Unguarded! + x += t_0 + t_1 + t_2_0 + t_2_1_0 + return x + + res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(res, 10 * TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_TOS_TUPLE", uops) + self.assertIn("_UNPACK_SEQUENCE_TUPLE", uops) + self.assertIn("_UNPACK_SEQUENCE_TWO_TUPLE", uops) + self.assertNotIn("_GUARD_NOS_TUPLE", uops) + self.assertIn("_BINARY_OP_SUBSCR_TUPLE_INT", uops) + def global_identity(x): return x From 0f643c489b02fe14a48186104513bf4e41df0cd2 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 8 Apr 2025 10:25:12 -0700 Subject: [PATCH 5/5] Fix backward tuple unpacking --- Python/optimizer_bytecodes.c | 2 +- Python/optimizer_cases.c.h | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 90d88c486278ac..07c44d7848fd7a 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -908,7 +908,7 @@ dummy_func(void) { op(_UNPACK_SEQUENCE_TUPLE, (seq -- values[oparg])) { for (int i = 0; i < oparg; i++) { - values[i] = sym_tuple_getitem(ctx, seq, i); + values[i] = sym_tuple_getitem(ctx, seq, oparg - i - 1); } } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 98eb078ca07900..e522157af72c20 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -218,7 +218,6 @@ JitOptSymbol *res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { - sym_set_type(value, &PyList_Type); res = sym_new_type(ctx, &PyBool_Type); } stack_pointer[-1] = res; @@ -873,7 +872,7 @@ seq = stack_pointer[-1]; values = &stack_pointer[-1]; for (int i = 0; i < oparg; i++) { - values[i] = sym_tuple_getitem(ctx, seq, i); + values[i] = sym_tuple_getitem(ctx, seq, oparg - i - 1); } stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS());