From 1d729b40bc96cd9c931cec3ce3e791b225ddaca9 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Tue, 20 Apr 2021 23:40:43 +1000 Subject: [PATCH 01/38] WIP bpo-43892: Make match patterns explicit in AST * Separates "pattern" and "expr" nodes in AST * AST node names are derived from the AST proposed in PEP 642, except that MatchValue always matches by equality and a separate MatchConstant node is defined for matching by identity * Grammar definition has been updated to emit the newly defined AST nodes * TODO: update code generator to consume new nodes * TODO: update AST validator to consume new nodes * TODO: update AST unparser to consume new nodes --- Grammar/python.gram | 135 +- Include/internal/pycore_ast.h | 110 +- Include/internal/pycore_ast_state.h | 11 + Parser/Python.asdl | 20 +- Parser/parser.c | 1767 ++++++++++++++++----------- Parser/pegen.c | 45 + Parser/pegen.h | 8 + Python/Python-ast.c | 1308 ++++++++++++++++---- 8 files changed, 2397 insertions(+), 1007 deletions(-) diff --git a/Grammar/python.gram b/Grammar/python.gram index d91e887e04dc00..9124a8a20b17b8 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -226,20 +226,20 @@ case_block[match_case_ty]: | invalid_case_block guard[expr_ty]: 'if' guard=named_expression { guard } -patterns[expr_ty]: - | values[asdl_expr_seq*]=open_sequence_pattern { - _PyAST_Tuple(values, Load, EXTRA) } +patterns[pattern_ty]: + | patterns[asdl_pattern_seq*]=open_sequence_pattern { + _PyAST_MatchSequence(patterns, EXTRA) } | pattern -pattern[expr_ty]: +pattern[pattern_ty]: | as_pattern | or_pattern -as_pattern[expr_ty]: - | pattern=or_pattern 'as' target=capture_pattern { +as_pattern[pattern_ty]: + | pattern=or_pattern 'as' target=pattern_capture_target { _PyAST_MatchAs(pattern, target->v.Name.id, EXTRA) } -or_pattern[expr_ty]: - | patterns[asdl_expr_seq*]='|'.closed_pattern+ { +or_pattern[pattern_ty]: + | patterns[asdl_pattern_seq*]='|'.closed_pattern+ { asdl_seq_LEN(patterns) == 1 ? asdl_seq_GET(patterns, 0) : _PyAST_MatchOr(patterns, EXTRA) } -closed_pattern[expr_ty]: +closed_pattern[pattern_ty]: | literal_pattern | capture_pattern | wildcard_pattern @@ -249,27 +249,44 @@ closed_pattern[expr_ty]: | mapping_pattern | class_pattern -literal_pattern[expr_ty]: +# Literal patterns are used for equality and identity constraints +literal_pattern[pattern_ty]: + | value=signed_number !('+' | '-') { _PyAST_MatchValue(value, EXTRA) } + | value=complex_number { _PyAST_MatchValue(value, EXTRA) } + | value=strings { _PyAST_MatchValue(value, EXTRA) } + | 'None' { _PyAST_MatchConstant(Py_None, EXTRA) } + | 'True' { _PyAST_MatchConstant(Py_True, EXTRA) } + | 'False' { _PyAST_MatchConstant(Py_False, EXTRA) } + +# Literal expressions are used to restrict permitted mapping pattern keys +literal_expr[expr_ty]: | signed_number !('+' | '-') - | real=signed_number '+' imag=NUMBER { _PyAST_BinOp(real, Add, imag, EXTRA) } - | real=signed_number '-' imag=NUMBER { _PyAST_BinOp(real, Sub, imag, EXTRA) } + | complex_number | strings | 'None' { _PyAST_Constant(Py_None, NULL, EXTRA) } | 'True' { _PyAST_Constant(Py_True, NULL, EXTRA) } | 'False' { _PyAST_Constant(Py_False, NULL, EXTRA) } + +complex_number[expr_ty]: + | real=signed_number '+' imag=NUMBER { _PyAST_BinOp(real, Add, imag, EXTRA) } + | real=signed_number '-' imag=NUMBER { _PyAST_BinOp(real, Sub, imag, EXTRA) } + signed_number[expr_ty]: | NUMBER | '-' number=NUMBER { _PyAST_UnaryOp(USub, number, EXTRA) } -capture_pattern[expr_ty]: +capture_pattern[pattern_ty]: + | target=pattern_capture_target { _PyAST_MatchAs(NULL, target->v.Name.id, EXTRA) } + +pattern_capture_target[expr_ty]: | !"_" name=NAME !('.' | '(' | '=') { _PyPegen_set_expr_context(p, name, Store) } -wildcard_pattern[expr_ty]: - | "_" { _PyAST_Name(CHECK(PyObject*, _PyPegen_new_identifier(p, "_")), Store, EXTRA) } +wildcard_pattern[pattern_ty]: + | "_" { _PyAST_MatchAlways(EXTRA) } -value_pattern[expr_ty]: - | attr=attr !('.' | '(' | '=') { attr } +value_pattern[pattern_ty]: + | attr=attr !('.' | '(' | '=') { _PyAST_MatchValue(attr, EXTRA) } attr[expr_ty]: | value=name_or_attr '.' attr=NAME { _PyAST_Attribute(value, attr->v.Name.id, Load, EXTRA) } @@ -277,50 +294,68 @@ name_or_attr[expr_ty]: | attr | NAME -group_pattern[expr_ty]: +group_pattern[pattern_ty]: | '(' pattern=pattern ')' { pattern } -sequence_pattern[expr_ty]: - | '[' values=maybe_sequence_pattern? ']' { _PyAST_List(values, Load, EXTRA) } - | '(' values=open_sequence_pattern? ')' { _PyAST_Tuple(values, Load, EXTRA) } +sequence_pattern[pattern_ty]: + | '[' patterns=maybe_sequence_pattern? ']' { _PyAST_MatchSequence(patterns, EXTRA) } + | '(' patterns=open_sequence_pattern? ')' { _PyAST_MatchSequence(patterns, EXTRA) } open_sequence_pattern[asdl_seq*]: - | value=maybe_star_pattern ',' values=maybe_sequence_pattern? { - _PyPegen_seq_insert_in_front(p, value, values) } + | pattern=maybe_star_pattern ',' patterns=maybe_sequence_pattern? { + _PyPegen_seq_insert_in_front(p, pattern, patterns) } maybe_sequence_pattern[asdl_seq*]: - | values=','.maybe_star_pattern+ ','? { values } -maybe_star_pattern[expr_ty]: + | patterns=','.maybe_star_pattern+ ','? { patterns } +maybe_star_pattern[pattern_ty]: | star_pattern | pattern -star_pattern[expr_ty]: - | '*' value=(capture_pattern | wildcard_pattern) { - _PyAST_Starred(value, Store, EXTRA) } +star_pattern[pattern_ty]: + | '*' target=pattern_capture_target { + _PyAST_MatchRestOfSequence(target->v.Name.id, EXTRA) } + | '*' wildcard_pattern { + _PyAST_MatchRestOfSequence(NULL, EXTRA) } -mapping_pattern[expr_ty]: +mapping_pattern[pattern_ty]: | '{' items=items_pattern? '}' { - _PyAST_Dict(CHECK(asdl_expr_seq*, _PyPegen_get_keys(p, items)), CHECK(asdl_expr_seq*, _PyPegen_get_values(p, items)), EXTRA) } + _PyAST_MatchMapping( + CHECK(asdl_expr_seq*, _PyPegen_get_pattern_keys(p, items)), + CHECK(asdl_pattern_seq*, _PyPegen_get_patterns(p, items)), + EXTRA) } items_pattern[asdl_seq*]: | items=','.key_value_pattern+ ','? { items } -key_value_pattern[KeyValuePair*]: - | key=(literal_pattern | value_pattern) ':' value=pattern { - _PyPegen_key_value_pair(p, key, value) } +key_value_pattern[KeyPatternPair*]: + | key=(literal_expr | attr) ':' pattern=pattern { + _PyPegen_key_pattern_pair(p, key, pattern) } | double_star_pattern -double_star_pattern[KeyValuePair*]: - | '**' value=capture_pattern { _PyPegen_key_value_pair(p, NULL, value) } - -class_pattern[expr_ty]: - | func=name_or_attr '(' ')' { _PyAST_Call(func, NULL, NULL, EXTRA) } - | func=name_or_attr '(' args=positional_patterns ','? ')' { - _PyAST_Call(func, args, NULL, EXTRA) } - | func=name_or_attr '(' keywords=keyword_patterns ','? ')' { - _PyAST_Call(func, NULL, keywords, EXTRA) } - | func=name_or_attr '(' args=positional_patterns ',' keywords=keyword_patterns ','? ')' { - _PyAST_Call(func, args, keywords, EXTRA) } -positional_patterns[asdl_expr_seq*]: - | args[asdl_expr_seq*]=','.pattern+ { args } -keyword_patterns[asdl_keyword_seq*]: - | keywords[asdl_keyword_seq*]=','.keyword_pattern+ { keywords } -keyword_pattern[keyword_ty]: - | arg=NAME '=' value=pattern { _PyAST_keyword(arg->v.Name.id, value, EXTRA) } +double_star_pattern[KeyPatternPair*]: + | '**' target=pattern_capture_target { + _PyPegen_key_pattern_pair(p, NULL, _PyAST_MatchAs(NULL, target->v.Name.id, EXTRA)) } + +class_pattern[pattern_ty]: + | cls=name_or_attr '(' ')' { + _PyAST_MatchClass(cls, NULL, NULL, NULL, EXTRA) } + | cls=name_or_attr '(' patterns=positional_patterns ','? ')' { + _PyAST_MatchClass(cls, patterns, NULL, NULL, EXTRA) } + | cls=name_or_attr '(' keywords=keyword_patterns ','? ')' { + _PyAST_MatchClass( + cls, NULL, + CHECK(asdl_identifier_seq*, _PyPegen_map_names_to_ids(p, + CHECK(asdl_expr_seq*, _PyPegen_get_pattern_keys(p, keywords)))), + CHECK(asdl_pattern_seq*, _PyPegen_get_patterns(p, keywords)), + EXTRA) } + | cls=name_or_attr '(' patterns=positional_patterns ',' keywords=keyword_patterns ','? ')' { + _PyAST_MatchClass( + cls, + patterns, + CHECK(asdl_identifier_seq*, _PyPegen_map_names_to_ids(p, + CHECK(asdl_expr_seq*, _PyPegen_get_pattern_keys(p, keywords)))), + CHECK(asdl_pattern_seq*, _PyPegen_get_patterns(p, keywords)), + EXTRA) } +positional_patterns[asdl_pattern_seq*]: + | args[asdl_pattern_seq*]=','.pattern+ { args } +keyword_patterns[asdl_seq*]: + | keywords[asdl_seq*]=','.keyword_pattern+ { keywords } +keyword_pattern[pattern_ty]: + | arg=NAME '=' value=pattern { _PyPegen_key_pattern_pair(p, arg, value) } return_stmt[stmt_ty]: | 'return' a=[star_expressions] { _PyAST_Return(a, EXTRA) } diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index 64d68b2b32d904..9b22cd3bddd4f7 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -47,6 +47,8 @@ typedef struct _withitem *withitem_ty; typedef struct _match_case *match_case_ty; +typedef struct _pattern *pattern_ty; + typedef struct _type_ignore *type_ignore_ty; @@ -130,6 +132,13 @@ typedef struct { asdl_match_case_seq *_Py_asdl_match_case_seq_new(Py_ssize_t size, PyArena *arena); +typedef struct { + _ASDL_SEQ_HEAD + pattern_ty typed_elements[1]; +} asdl_pattern_seq; + +asdl_pattern_seq *_Py_asdl_pattern_seq_new(Py_ssize_t size, PyArena *arena); + typedef struct { _ASDL_SEQ_HEAD type_ignore_ty typed_elements[1]; @@ -327,8 +336,7 @@ enum _expr_kind {BoolOp_kind=1, NamedExpr_kind=2, BinOp_kind=3, UnaryOp_kind=4, YieldFrom_kind=15, Compare_kind=16, Call_kind=17, FormattedValue_kind=18, JoinedStr_kind=19, Constant_kind=20, Attribute_kind=21, Subscript_kind=22, Starred_kind=23, - Name_kind=24, List_kind=25, Tuple_kind=26, Slice_kind=27, - MatchAs_kind=28, MatchOr_kind=29}; + Name_kind=24, List_kind=25, Tuple_kind=26, Slice_kind=27}; struct _expr { enum _expr_kind kind; union { @@ -471,15 +479,6 @@ struct _expr { expr_ty step; } Slice; - struct { - expr_ty pattern; - identifier name; - } MatchAs; - - struct { - asdl_expr_seq *patterns; - } MatchOr; - } v; int lineno; int col_offset; @@ -555,11 +554,63 @@ struct _withitem { }; struct _match_case { - expr_ty pattern; + pattern_ty pattern; expr_ty guard; asdl_stmt_seq *body; }; +enum _pattern_kind {MatchAlways_kind=1, MatchValue_kind=2, + MatchConstant_kind=3, MatchSequence_kind=4, + MatchMapping_kind=5, MatchClass_kind=6, + MatchRestOfSequence_kind=7, MatchAs_kind=8, + MatchOr_kind=9}; +struct _pattern { + enum _pattern_kind kind; + union { + struct { + expr_ty value; + } MatchValue; + + struct { + constant value; + } MatchConstant; + + struct { + asdl_pattern_seq *patterns; + } MatchSequence; + + struct { + asdl_expr_seq *keys; + asdl_pattern_seq *patterns; + } MatchMapping; + + struct { + expr_ty cls; + asdl_pattern_seq *patterns; + asdl_identifier_seq *extra_attrs; + asdl_pattern_seq *extra_patterns; + } MatchClass; + + struct { + identifier target; + } MatchRestOfSequence; + + struct { + pattern_ty pattern; + identifier target; + } MatchAs; + + struct { + asdl_pattern_seq *patterns; + } MatchOr; + + } v; + int lineno; + int col_offset; + int end_lineno; + int end_col_offset; +}; + enum _type_ignore_kind {TypeIgnore_kind=1}; struct _type_ignore { enum _type_ignore_kind kind; @@ -733,11 +784,6 @@ expr_ty _PyAST_Tuple(asdl_expr_seq * elts, expr_context_ty ctx, int lineno, int expr_ty _PyAST_Slice(expr_ty lower, expr_ty upper, expr_ty step, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); -expr_ty _PyAST_MatchAs(expr_ty pattern, identifier name, int lineno, int - col_offset, int end_lineno, int end_col_offset, PyArena - *arena); -expr_ty _PyAST_MatchOr(asdl_expr_seq * patterns, int lineno, int col_offset, - int end_lineno, int end_col_offset, PyArena *arena); comprehension_ty _PyAST_comprehension(expr_ty target, expr_ty iter, asdl_expr_seq * ifs, int is_async, PyArena *arena); @@ -760,8 +806,34 @@ alias_ty _PyAST_alias(identifier name, identifier asname, int lineno, int *arena); withitem_ty _PyAST_withitem(expr_ty context_expr, expr_ty optional_vars, PyArena *arena); -match_case_ty _PyAST_match_case(expr_ty pattern, expr_ty guard, asdl_stmt_seq * - body, PyArena *arena); +match_case_ty _PyAST_match_case(pattern_ty pattern, expr_ty guard, + asdl_stmt_seq * body, PyArena *arena); +pattern_ty _PyAST_MatchAlways(int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +pattern_ty _PyAST_MatchValue(expr_ty value, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +pattern_ty _PyAST_MatchConstant(constant value, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +pattern_ty _PyAST_MatchSequence(asdl_pattern_seq * patterns, int lineno, int + col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +pattern_ty _PyAST_MatchMapping(asdl_expr_seq * keys, asdl_pattern_seq * + patterns, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +pattern_ty _PyAST_MatchClass(expr_ty cls, asdl_pattern_seq * patterns, + asdl_identifier_seq * extra_attrs, + asdl_pattern_seq * extra_patterns, int lineno, int + col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +pattern_ty _PyAST_MatchRestOfSequence(identifier target, int lineno, int + col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +pattern_ty _PyAST_MatchAs(pattern_ty pattern, identifier target, int lineno, + int col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +pattern_ty _PyAST_MatchOr(asdl_pattern_seq * patterns, int lineno, int + col_offset, int end_lineno, int end_col_offset, + PyArena *arena); type_ignore_ty _PyAST_TypeIgnore(int lineno, string tag, PyArena *arena); diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index c5ae2242177920..457e51c4b08c4e 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -91,8 +91,15 @@ struct ast_state { PyObject *Lt_type; PyObject *MatMult_singleton; PyObject *MatMult_type; + PyObject *MatchAlways_type; PyObject *MatchAs_type; + PyObject *MatchClass_type; + PyObject *MatchConstant_type; + PyObject *MatchMapping_type; PyObject *MatchOr_type; + PyObject *MatchRestOfSequence_type; + PyObject *MatchSequence_type; + PyObject *MatchValue_type; PyObject *Match_type; PyObject *Mod_singleton; PyObject *Mod_type; @@ -159,6 +166,7 @@ struct ast_state { PyObject *boolop_type; PyObject *cases; PyObject *cause; + PyObject *cls; PyObject *cmpop_type; PyObject *col_offset; PyObject *comparators; @@ -176,6 +184,8 @@ struct ast_state { PyObject *excepthandler_type; PyObject *expr_context_type; PyObject *expr_type; + PyObject *extra_attrs; + PyObject *extra_patterns; PyObject *finalbody; PyObject *format_spec; PyObject *func; @@ -212,6 +222,7 @@ struct ast_state { PyObject *optional_vars; PyObject *orelse; PyObject *pattern; + PyObject *pattern_type; PyObject *patterns; PyObject *posonlyargs; PyObject *returns; diff --git a/Parser/Python.asdl b/Parser/Python.asdl index e224f5f4848dc1..5b98cdf5b8751e 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -89,10 +89,6 @@ module Python -- can appear only in Subscript | Slice(expr? lower, expr? upper, expr? step) - -- only used in patterns - | MatchAs(expr pattern, identifier name) - | MatchOr(expr* patterns) - -- col_offset is the byte offset in the utf8 string the parser uses attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) @@ -128,7 +124,21 @@ module Python withitem = (expr context_expr, expr? optional_vars) - match_case = (expr pattern, expr? guard, stmt* body) + match_case = (pattern pattern, expr? guard, stmt* body) + + pattern = MatchAlways + | MatchValue(expr value) + | MatchConstant(constant value) + | MatchSequence(pattern* patterns) + | MatchMapping(expr* keys, pattern* patterns) + | MatchClass(expr cls, pattern* patterns, identifier* extra_attrs, pattern* extra_patterns) + + | MatchRestOfSequence(identifier? target) + -- A NULL entry in the MatchMapping key list handles capturing extra mapping keys + + | MatchAs(pattern? pattern, identifier target) + | MatchOr(pattern* patterns) + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) type_ignore = TypeIgnore(int lineno, string tag) } diff --git a/Parser/parser.c b/Parser/parser.c index e47093e1dca7dd..70963b0ce57dfa 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -119,353 +119,356 @@ static char *soft_keywords[] = { #define or_pattern_type 1045 #define closed_pattern_type 1046 #define literal_pattern_type 1047 -#define signed_number_type 1048 -#define capture_pattern_type 1049 -#define wildcard_pattern_type 1050 -#define value_pattern_type 1051 -#define attr_type 1052 // Left-recursive -#define name_or_attr_type 1053 // Left-recursive -#define group_pattern_type 1054 -#define sequence_pattern_type 1055 -#define open_sequence_pattern_type 1056 -#define maybe_sequence_pattern_type 1057 -#define maybe_star_pattern_type 1058 -#define star_pattern_type 1059 -#define mapping_pattern_type 1060 -#define items_pattern_type 1061 -#define key_value_pattern_type 1062 -#define double_star_pattern_type 1063 -#define class_pattern_type 1064 -#define positional_patterns_type 1065 -#define keyword_patterns_type 1066 -#define keyword_pattern_type 1067 -#define return_stmt_type 1068 -#define raise_stmt_type 1069 -#define function_def_type 1070 -#define function_def_raw_type 1071 -#define func_type_comment_type 1072 -#define params_type 1073 -#define parameters_type 1074 -#define slash_no_default_type 1075 -#define slash_with_default_type 1076 -#define star_etc_type 1077 -#define kwds_type 1078 -#define param_no_default_type 1079 -#define param_with_default_type 1080 -#define param_maybe_default_type 1081 -#define param_type 1082 -#define annotation_type 1083 -#define default_type 1084 -#define decorators_type 1085 -#define class_def_type 1086 -#define class_def_raw_type 1087 -#define block_type 1088 -#define star_expressions_type 1089 -#define star_expression_type 1090 -#define star_named_expressions_type 1091 -#define star_named_expression_type 1092 -#define named_expression_type 1093 -#define direct_named_expression_type 1094 -#define annotated_rhs_type 1095 -#define expressions_type 1096 -#define expression_type 1097 -#define lambdef_type 1098 -#define lambda_params_type 1099 -#define lambda_parameters_type 1100 -#define lambda_slash_no_default_type 1101 -#define lambda_slash_with_default_type 1102 -#define lambda_star_etc_type 1103 -#define lambda_kwds_type 1104 -#define lambda_param_no_default_type 1105 -#define lambda_param_with_default_type 1106 -#define lambda_param_maybe_default_type 1107 -#define lambda_param_type 1108 -#define disjunction_type 1109 -#define conjunction_type 1110 -#define inversion_type 1111 -#define comparison_type 1112 -#define compare_op_bitwise_or_pair_type 1113 -#define eq_bitwise_or_type 1114 -#define noteq_bitwise_or_type 1115 -#define lte_bitwise_or_type 1116 -#define lt_bitwise_or_type 1117 -#define gte_bitwise_or_type 1118 -#define gt_bitwise_or_type 1119 -#define notin_bitwise_or_type 1120 -#define in_bitwise_or_type 1121 -#define isnot_bitwise_or_type 1122 -#define is_bitwise_or_type 1123 -#define bitwise_or_type 1124 // Left-recursive -#define bitwise_xor_type 1125 // Left-recursive -#define bitwise_and_type 1126 // Left-recursive -#define shift_expr_type 1127 // Left-recursive -#define sum_type 1128 // Left-recursive -#define term_type 1129 // Left-recursive -#define factor_type 1130 -#define power_type 1131 -#define await_primary_type 1132 -#define primary_type 1133 // Left-recursive -#define slices_type 1134 -#define slice_type 1135 -#define atom_type 1136 -#define strings_type 1137 -#define list_type 1138 -#define listcomp_type 1139 -#define tuple_type 1140 -#define group_type 1141 -#define genexp_type 1142 -#define set_type 1143 -#define setcomp_type 1144 -#define dict_type 1145 -#define dictcomp_type 1146 -#define double_starred_kvpairs_type 1147 -#define double_starred_kvpair_type 1148 -#define kvpair_type 1149 -#define for_if_clauses_type 1150 -#define for_if_clause_type 1151 -#define yield_expr_type 1152 -#define arguments_type 1153 -#define args_type 1154 -#define kwargs_type 1155 -#define starred_expression_type 1156 -#define kwarg_or_starred_type 1157 -#define kwarg_or_double_starred_type 1158 -#define star_targets_type 1159 -#define star_targets_list_seq_type 1160 -#define star_targets_tuple_seq_type 1161 -#define star_target_type 1162 -#define target_with_star_atom_type 1163 -#define star_atom_type 1164 -#define single_target_type 1165 -#define single_subscript_attribute_target_type 1166 -#define del_targets_type 1167 -#define del_target_type 1168 -#define del_t_atom_type 1169 -#define targets_type 1170 -#define target_type 1171 -#define t_primary_type 1172 // Left-recursive -#define t_lookahead_type 1173 -#define t_atom_type 1174 -#define invalid_arguments_type 1175 -#define invalid_kwarg_type 1176 -#define invalid_expression_type 1177 -#define invalid_named_expression_type 1178 -#define invalid_assignment_type 1179 -#define invalid_ann_assign_target_type 1180 -#define invalid_del_stmt_type 1181 -#define invalid_block_type 1182 -#define invalid_primary_type 1183 // Left-recursive -#define invalid_comprehension_type 1184 -#define invalid_dict_comprehension_type 1185 -#define invalid_parameters_type 1186 -#define invalid_parameters_helper_type 1187 -#define invalid_lambda_parameters_type 1188 -#define invalid_lambda_parameters_helper_type 1189 -#define invalid_star_etc_type 1190 -#define invalid_lambda_star_etc_type 1191 -#define invalid_double_type_comments_type 1192 -#define invalid_with_item_type 1193 -#define invalid_for_target_type 1194 -#define invalid_group_type 1195 -#define invalid_import_from_targets_type 1196 -#define invalid_with_stmt_type 1197 -#define invalid_except_block_type 1198 -#define invalid_match_stmt_type 1199 -#define invalid_case_block_type 1200 -#define invalid_if_stmt_type 1201 -#define invalid_elif_stmt_type 1202 -#define invalid_while_stmt_type 1203 -#define invalid_double_starred_kvpairs_type 1204 -#define invalid_kvpair_type 1205 -#define _loop0_1_type 1206 -#define _loop0_2_type 1207 -#define _loop0_4_type 1208 -#define _gather_3_type 1209 -#define _loop0_6_type 1210 -#define _gather_5_type 1211 -#define _loop0_8_type 1212 -#define _gather_7_type 1213 -#define _loop0_10_type 1214 -#define _gather_9_type 1215 -#define _loop1_11_type 1216 -#define _loop0_13_type 1217 -#define _gather_12_type 1218 -#define _tmp_14_type 1219 -#define _tmp_15_type 1220 -#define _tmp_16_type 1221 -#define _tmp_17_type 1222 -#define _tmp_18_type 1223 -#define _tmp_19_type 1224 -#define _tmp_20_type 1225 -#define _tmp_21_type 1226 -#define _loop1_22_type 1227 -#define _tmp_23_type 1228 -#define _tmp_24_type 1229 -#define _loop0_26_type 1230 -#define _gather_25_type 1231 -#define _loop0_28_type 1232 -#define _gather_27_type 1233 -#define _tmp_29_type 1234 -#define _tmp_30_type 1235 -#define _loop0_31_type 1236 -#define _loop1_32_type 1237 -#define _loop0_34_type 1238 -#define _gather_33_type 1239 -#define _tmp_35_type 1240 -#define _loop0_37_type 1241 -#define _gather_36_type 1242 -#define _tmp_38_type 1243 -#define _loop0_40_type 1244 -#define _gather_39_type 1245 -#define _loop0_42_type 1246 -#define _gather_41_type 1247 -#define _loop0_44_type 1248 -#define _gather_43_type 1249 -#define _loop0_46_type 1250 -#define _gather_45_type 1251 -#define _tmp_47_type 1252 -#define _loop1_48_type 1253 -#define _tmp_49_type 1254 -#define _loop1_50_type 1255 -#define _loop0_52_type 1256 -#define _gather_51_type 1257 -#define _tmp_53_type 1258 -#define _tmp_54_type 1259 -#define _tmp_55_type 1260 -#define _loop0_57_type 1261 -#define _gather_56_type 1262 -#define _tmp_58_type 1263 -#define _loop0_60_type 1264 -#define _gather_59_type 1265 -#define _tmp_61_type 1266 -#define _loop0_63_type 1267 -#define _gather_62_type 1268 -#define _loop0_65_type 1269 -#define _gather_64_type 1270 -#define _tmp_66_type 1271 -#define _tmp_67_type 1272 -#define _tmp_68_type 1273 -#define _tmp_69_type 1274 -#define _loop0_70_type 1275 -#define _loop0_71_type 1276 -#define _loop0_72_type 1277 -#define _loop1_73_type 1278 -#define _loop0_74_type 1279 -#define _loop1_75_type 1280 -#define _loop1_76_type 1281 -#define _loop1_77_type 1282 -#define _loop0_78_type 1283 -#define _loop1_79_type 1284 -#define _loop0_80_type 1285 -#define _loop1_81_type 1286 -#define _loop0_82_type 1287 -#define _loop1_83_type 1288 -#define _loop1_84_type 1289 -#define _tmp_85_type 1290 -#define _loop1_86_type 1291 -#define _loop0_88_type 1292 -#define _gather_87_type 1293 -#define _loop1_89_type 1294 -#define _loop0_90_type 1295 -#define _loop0_91_type 1296 -#define _loop0_92_type 1297 -#define _loop1_93_type 1298 -#define _loop0_94_type 1299 -#define _loop1_95_type 1300 -#define _loop1_96_type 1301 -#define _loop1_97_type 1302 -#define _loop0_98_type 1303 -#define _loop1_99_type 1304 -#define _loop0_100_type 1305 -#define _loop1_101_type 1306 -#define _loop0_102_type 1307 -#define _loop1_103_type 1308 -#define _loop1_104_type 1309 -#define _loop1_105_type 1310 -#define _loop1_106_type 1311 -#define _tmp_107_type 1312 -#define _loop0_109_type 1313 -#define _gather_108_type 1314 -#define _tmp_110_type 1315 -#define _tmp_111_type 1316 -#define _tmp_112_type 1317 -#define _tmp_113_type 1318 -#define _loop1_114_type 1319 -#define _tmp_115_type 1320 -#define _tmp_116_type 1321 -#define _loop0_118_type 1322 -#define _gather_117_type 1323 -#define _loop1_119_type 1324 -#define _loop0_120_type 1325 -#define _loop0_121_type 1326 -#define _loop0_123_type 1327 -#define _gather_122_type 1328 -#define _tmp_124_type 1329 -#define _loop0_126_type 1330 -#define _gather_125_type 1331 -#define _loop0_128_type 1332 -#define _gather_127_type 1333 -#define _loop0_130_type 1334 -#define _gather_129_type 1335 -#define _loop0_132_type 1336 -#define _gather_131_type 1337 -#define _loop0_133_type 1338 -#define _loop0_135_type 1339 -#define _gather_134_type 1340 -#define _loop1_136_type 1341 -#define _tmp_137_type 1342 -#define _loop0_139_type 1343 -#define _gather_138_type 1344 -#define _loop0_141_type 1345 -#define _gather_140_type 1346 -#define _tmp_142_type 1347 -#define _tmp_143_type 1348 -#define _tmp_144_type 1349 -#define _tmp_145_type 1350 -#define _tmp_146_type 1351 -#define _loop0_147_type 1352 -#define _loop0_148_type 1353 -#define _loop0_149_type 1354 -#define _tmp_150_type 1355 -#define _tmp_151_type 1356 -#define _tmp_152_type 1357 -#define _loop0_153_type 1358 -#define _loop1_154_type 1359 -#define _loop0_155_type 1360 -#define _loop1_156_type 1361 -#define _tmp_157_type 1362 -#define _tmp_158_type 1363 -#define _tmp_159_type 1364 -#define _loop0_161_type 1365 -#define _gather_160_type 1366 -#define _loop0_163_type 1367 -#define _gather_162_type 1368 -#define _tmp_164_type 1369 -#define _tmp_165_type 1370 -#define _loop0_167_type 1371 -#define _gather_166_type 1372 -#define _tmp_168_type 1373 -#define _tmp_169_type 1374 -#define _tmp_170_type 1375 -#define _tmp_171_type 1376 -#define _tmp_172_type 1377 -#define _tmp_173_type 1378 -#define _tmp_174_type 1379 -#define _tmp_175_type 1380 -#define _tmp_176_type 1381 -#define _tmp_177_type 1382 -#define _tmp_178_type 1383 -#define _tmp_179_type 1384 -#define _tmp_180_type 1385 -#define _tmp_181_type 1386 -#define _tmp_182_type 1387 -#define _tmp_183_type 1388 -#define _tmp_184_type 1389 -#define _tmp_185_type 1390 -#define _tmp_186_type 1391 -#define _tmp_187_type 1392 -#define _tmp_188_type 1393 -#define _tmp_189_type 1394 +#define literal_expr_type 1048 +#define complex_number_type 1049 +#define signed_number_type 1050 +#define capture_pattern_type 1051 +#define pattern_capture_target_type 1052 +#define wildcard_pattern_type 1053 +#define value_pattern_type 1054 +#define attr_type 1055 // Left-recursive +#define name_or_attr_type 1056 // Left-recursive +#define group_pattern_type 1057 +#define sequence_pattern_type 1058 +#define open_sequence_pattern_type 1059 +#define maybe_sequence_pattern_type 1060 +#define maybe_star_pattern_type 1061 +#define star_pattern_type 1062 +#define mapping_pattern_type 1063 +#define items_pattern_type 1064 +#define key_value_pattern_type 1065 +#define double_star_pattern_type 1066 +#define class_pattern_type 1067 +#define positional_patterns_type 1068 +#define keyword_patterns_type 1069 +#define keyword_pattern_type 1070 +#define return_stmt_type 1071 +#define raise_stmt_type 1072 +#define function_def_type 1073 +#define function_def_raw_type 1074 +#define func_type_comment_type 1075 +#define params_type 1076 +#define parameters_type 1077 +#define slash_no_default_type 1078 +#define slash_with_default_type 1079 +#define star_etc_type 1080 +#define kwds_type 1081 +#define param_no_default_type 1082 +#define param_with_default_type 1083 +#define param_maybe_default_type 1084 +#define param_type 1085 +#define annotation_type 1086 +#define default_type 1087 +#define decorators_type 1088 +#define class_def_type 1089 +#define class_def_raw_type 1090 +#define block_type 1091 +#define star_expressions_type 1092 +#define star_expression_type 1093 +#define star_named_expressions_type 1094 +#define star_named_expression_type 1095 +#define named_expression_type 1096 +#define direct_named_expression_type 1097 +#define annotated_rhs_type 1098 +#define expressions_type 1099 +#define expression_type 1100 +#define lambdef_type 1101 +#define lambda_params_type 1102 +#define lambda_parameters_type 1103 +#define lambda_slash_no_default_type 1104 +#define lambda_slash_with_default_type 1105 +#define lambda_star_etc_type 1106 +#define lambda_kwds_type 1107 +#define lambda_param_no_default_type 1108 +#define lambda_param_with_default_type 1109 +#define lambda_param_maybe_default_type 1110 +#define lambda_param_type 1111 +#define disjunction_type 1112 +#define conjunction_type 1113 +#define inversion_type 1114 +#define comparison_type 1115 +#define compare_op_bitwise_or_pair_type 1116 +#define eq_bitwise_or_type 1117 +#define noteq_bitwise_or_type 1118 +#define lte_bitwise_or_type 1119 +#define lt_bitwise_or_type 1120 +#define gte_bitwise_or_type 1121 +#define gt_bitwise_or_type 1122 +#define notin_bitwise_or_type 1123 +#define in_bitwise_or_type 1124 +#define isnot_bitwise_or_type 1125 +#define is_bitwise_or_type 1126 +#define bitwise_or_type 1127 // Left-recursive +#define bitwise_xor_type 1128 // Left-recursive +#define bitwise_and_type 1129 // Left-recursive +#define shift_expr_type 1130 // Left-recursive +#define sum_type 1131 // Left-recursive +#define term_type 1132 // Left-recursive +#define factor_type 1133 +#define power_type 1134 +#define await_primary_type 1135 +#define primary_type 1136 // Left-recursive +#define slices_type 1137 +#define slice_type 1138 +#define atom_type 1139 +#define strings_type 1140 +#define list_type 1141 +#define listcomp_type 1142 +#define tuple_type 1143 +#define group_type 1144 +#define genexp_type 1145 +#define set_type 1146 +#define setcomp_type 1147 +#define dict_type 1148 +#define dictcomp_type 1149 +#define double_starred_kvpairs_type 1150 +#define double_starred_kvpair_type 1151 +#define kvpair_type 1152 +#define for_if_clauses_type 1153 +#define for_if_clause_type 1154 +#define yield_expr_type 1155 +#define arguments_type 1156 +#define args_type 1157 +#define kwargs_type 1158 +#define starred_expression_type 1159 +#define kwarg_or_starred_type 1160 +#define kwarg_or_double_starred_type 1161 +#define star_targets_type 1162 +#define star_targets_list_seq_type 1163 +#define star_targets_tuple_seq_type 1164 +#define star_target_type 1165 +#define target_with_star_atom_type 1166 +#define star_atom_type 1167 +#define single_target_type 1168 +#define single_subscript_attribute_target_type 1169 +#define del_targets_type 1170 +#define del_target_type 1171 +#define del_t_atom_type 1172 +#define targets_type 1173 +#define target_type 1174 +#define t_primary_type 1175 // Left-recursive +#define t_lookahead_type 1176 +#define t_atom_type 1177 +#define invalid_arguments_type 1178 +#define invalid_kwarg_type 1179 +#define invalid_expression_type 1180 +#define invalid_named_expression_type 1181 +#define invalid_assignment_type 1182 +#define invalid_ann_assign_target_type 1183 +#define invalid_del_stmt_type 1184 +#define invalid_block_type 1185 +#define invalid_primary_type 1186 // Left-recursive +#define invalid_comprehension_type 1187 +#define invalid_dict_comprehension_type 1188 +#define invalid_parameters_type 1189 +#define invalid_parameters_helper_type 1190 +#define invalid_lambda_parameters_type 1191 +#define invalid_lambda_parameters_helper_type 1192 +#define invalid_star_etc_type 1193 +#define invalid_lambda_star_etc_type 1194 +#define invalid_double_type_comments_type 1195 +#define invalid_with_item_type 1196 +#define invalid_for_target_type 1197 +#define invalid_group_type 1198 +#define invalid_import_from_targets_type 1199 +#define invalid_with_stmt_type 1200 +#define invalid_except_block_type 1201 +#define invalid_match_stmt_type 1202 +#define invalid_case_block_type 1203 +#define invalid_if_stmt_type 1204 +#define invalid_elif_stmt_type 1205 +#define invalid_while_stmt_type 1206 +#define invalid_double_starred_kvpairs_type 1207 +#define invalid_kvpair_type 1208 +#define _loop0_1_type 1209 +#define _loop0_2_type 1210 +#define _loop0_4_type 1211 +#define _gather_3_type 1212 +#define _loop0_6_type 1213 +#define _gather_5_type 1214 +#define _loop0_8_type 1215 +#define _gather_7_type 1216 +#define _loop0_10_type 1217 +#define _gather_9_type 1218 +#define _loop1_11_type 1219 +#define _loop0_13_type 1220 +#define _gather_12_type 1221 +#define _tmp_14_type 1222 +#define _tmp_15_type 1223 +#define _tmp_16_type 1224 +#define _tmp_17_type 1225 +#define _tmp_18_type 1226 +#define _tmp_19_type 1227 +#define _tmp_20_type 1228 +#define _tmp_21_type 1229 +#define _loop1_22_type 1230 +#define _tmp_23_type 1231 +#define _tmp_24_type 1232 +#define _loop0_26_type 1233 +#define _gather_25_type 1234 +#define _loop0_28_type 1235 +#define _gather_27_type 1236 +#define _tmp_29_type 1237 +#define _tmp_30_type 1238 +#define _loop0_31_type 1239 +#define _loop1_32_type 1240 +#define _loop0_34_type 1241 +#define _gather_33_type 1242 +#define _tmp_35_type 1243 +#define _loop0_37_type 1244 +#define _gather_36_type 1245 +#define _tmp_38_type 1246 +#define _loop0_40_type 1247 +#define _gather_39_type 1248 +#define _loop0_42_type 1249 +#define _gather_41_type 1250 +#define _loop0_44_type 1251 +#define _gather_43_type 1252 +#define _loop0_46_type 1253 +#define _gather_45_type 1254 +#define _tmp_47_type 1255 +#define _loop1_48_type 1256 +#define _tmp_49_type 1257 +#define _loop1_50_type 1258 +#define _loop0_52_type 1259 +#define _gather_51_type 1260 +#define _tmp_53_type 1261 +#define _tmp_54_type 1262 +#define _tmp_55_type 1263 +#define _tmp_56_type 1264 +#define _loop0_58_type 1265 +#define _gather_57_type 1266 +#define _loop0_60_type 1267 +#define _gather_59_type 1268 +#define _tmp_61_type 1269 +#define _loop0_63_type 1270 +#define _gather_62_type 1271 +#define _loop0_65_type 1272 +#define _gather_64_type 1273 +#define _tmp_66_type 1274 +#define _tmp_67_type 1275 +#define _tmp_68_type 1276 +#define _tmp_69_type 1277 +#define _loop0_70_type 1278 +#define _loop0_71_type 1279 +#define _loop0_72_type 1280 +#define _loop1_73_type 1281 +#define _loop0_74_type 1282 +#define _loop1_75_type 1283 +#define _loop1_76_type 1284 +#define _loop1_77_type 1285 +#define _loop0_78_type 1286 +#define _loop1_79_type 1287 +#define _loop0_80_type 1288 +#define _loop1_81_type 1289 +#define _loop0_82_type 1290 +#define _loop1_83_type 1291 +#define _loop1_84_type 1292 +#define _tmp_85_type 1293 +#define _loop1_86_type 1294 +#define _loop0_88_type 1295 +#define _gather_87_type 1296 +#define _loop1_89_type 1297 +#define _loop0_90_type 1298 +#define _loop0_91_type 1299 +#define _loop0_92_type 1300 +#define _loop1_93_type 1301 +#define _loop0_94_type 1302 +#define _loop1_95_type 1303 +#define _loop1_96_type 1304 +#define _loop1_97_type 1305 +#define _loop0_98_type 1306 +#define _loop1_99_type 1307 +#define _loop0_100_type 1308 +#define _loop1_101_type 1309 +#define _loop0_102_type 1310 +#define _loop1_103_type 1311 +#define _loop1_104_type 1312 +#define _loop1_105_type 1313 +#define _loop1_106_type 1314 +#define _tmp_107_type 1315 +#define _loop0_109_type 1316 +#define _gather_108_type 1317 +#define _tmp_110_type 1318 +#define _tmp_111_type 1319 +#define _tmp_112_type 1320 +#define _tmp_113_type 1321 +#define _loop1_114_type 1322 +#define _tmp_115_type 1323 +#define _tmp_116_type 1324 +#define _loop0_118_type 1325 +#define _gather_117_type 1326 +#define _loop1_119_type 1327 +#define _loop0_120_type 1328 +#define _loop0_121_type 1329 +#define _loop0_123_type 1330 +#define _gather_122_type 1331 +#define _tmp_124_type 1332 +#define _loop0_126_type 1333 +#define _gather_125_type 1334 +#define _loop0_128_type 1335 +#define _gather_127_type 1336 +#define _loop0_130_type 1337 +#define _gather_129_type 1338 +#define _loop0_132_type 1339 +#define _gather_131_type 1340 +#define _loop0_133_type 1341 +#define _loop0_135_type 1342 +#define _gather_134_type 1343 +#define _loop1_136_type 1344 +#define _tmp_137_type 1345 +#define _loop0_139_type 1346 +#define _gather_138_type 1347 +#define _loop0_141_type 1348 +#define _gather_140_type 1349 +#define _tmp_142_type 1350 +#define _tmp_143_type 1351 +#define _tmp_144_type 1352 +#define _tmp_145_type 1353 +#define _tmp_146_type 1354 +#define _loop0_147_type 1355 +#define _loop0_148_type 1356 +#define _loop0_149_type 1357 +#define _tmp_150_type 1358 +#define _tmp_151_type 1359 +#define _tmp_152_type 1360 +#define _loop0_153_type 1361 +#define _loop1_154_type 1362 +#define _loop0_155_type 1363 +#define _loop1_156_type 1364 +#define _tmp_157_type 1365 +#define _tmp_158_type 1366 +#define _tmp_159_type 1367 +#define _loop0_161_type 1368 +#define _gather_160_type 1369 +#define _loop0_163_type 1370 +#define _gather_162_type 1371 +#define _tmp_164_type 1372 +#define _tmp_165_type 1373 +#define _loop0_167_type 1374 +#define _gather_166_type 1375 +#define _tmp_168_type 1376 +#define _tmp_169_type 1377 +#define _tmp_170_type 1378 +#define _tmp_171_type 1379 +#define _tmp_172_type 1380 +#define _tmp_173_type 1381 +#define _tmp_174_type 1382 +#define _tmp_175_type 1383 +#define _tmp_176_type 1384 +#define _tmp_177_type 1385 +#define _tmp_178_type 1386 +#define _tmp_179_type 1387 +#define _tmp_180_type 1388 +#define _tmp_181_type 1389 +#define _tmp_182_type 1390 +#define _tmp_183_type 1391 +#define _tmp_184_type 1392 +#define _tmp_185_type 1393 +#define _tmp_186_type 1394 +#define _tmp_187_type 1395 +#define _tmp_188_type 1396 +#define _tmp_189_type 1397 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -509,32 +512,35 @@ static stmt_ty match_stmt_rule(Parser *p); static expr_ty subject_expr_rule(Parser *p); static match_case_ty case_block_rule(Parser *p); static expr_ty guard_rule(Parser *p); -static expr_ty patterns_rule(Parser *p); -static expr_ty pattern_rule(Parser *p); -static expr_ty as_pattern_rule(Parser *p); -static expr_ty or_pattern_rule(Parser *p); -static expr_ty closed_pattern_rule(Parser *p); -static expr_ty literal_pattern_rule(Parser *p); +static pattern_ty patterns_rule(Parser *p); +static pattern_ty pattern_rule(Parser *p); +static pattern_ty as_pattern_rule(Parser *p); +static pattern_ty or_pattern_rule(Parser *p); +static pattern_ty closed_pattern_rule(Parser *p); +static pattern_ty literal_pattern_rule(Parser *p); +static expr_ty literal_expr_rule(Parser *p); +static expr_ty complex_number_rule(Parser *p); static expr_ty signed_number_rule(Parser *p); -static expr_ty capture_pattern_rule(Parser *p); -static expr_ty wildcard_pattern_rule(Parser *p); -static expr_ty value_pattern_rule(Parser *p); +static pattern_ty capture_pattern_rule(Parser *p); +static expr_ty pattern_capture_target_rule(Parser *p); +static pattern_ty wildcard_pattern_rule(Parser *p); +static pattern_ty value_pattern_rule(Parser *p); static expr_ty attr_rule(Parser *p); static expr_ty name_or_attr_rule(Parser *p); -static expr_ty group_pattern_rule(Parser *p); -static expr_ty sequence_pattern_rule(Parser *p); +static pattern_ty group_pattern_rule(Parser *p); +static pattern_ty sequence_pattern_rule(Parser *p); static asdl_seq* open_sequence_pattern_rule(Parser *p); static asdl_seq* maybe_sequence_pattern_rule(Parser *p); -static expr_ty maybe_star_pattern_rule(Parser *p); -static expr_ty star_pattern_rule(Parser *p); -static expr_ty mapping_pattern_rule(Parser *p); +static pattern_ty maybe_star_pattern_rule(Parser *p); +static pattern_ty star_pattern_rule(Parser *p); +static pattern_ty mapping_pattern_rule(Parser *p); static asdl_seq* items_pattern_rule(Parser *p); -static KeyValuePair* key_value_pattern_rule(Parser *p); -static KeyValuePair* double_star_pattern_rule(Parser *p); -static expr_ty class_pattern_rule(Parser *p); -static asdl_expr_seq* positional_patterns_rule(Parser *p); -static asdl_keyword_seq* keyword_patterns_rule(Parser *p); -static keyword_ty keyword_pattern_rule(Parser *p); +static KeyPatternPair* key_value_pattern_rule(Parser *p); +static KeyPatternPair* double_star_pattern_rule(Parser *p); +static pattern_ty class_pattern_rule(Parser *p); +static asdl_pattern_seq* positional_patterns_rule(Parser *p); +static asdl_seq* keyword_patterns_rule(Parser *p); +static pattern_ty keyword_pattern_rule(Parser *p); static stmt_ty return_stmt_rule(Parser *p); static stmt_ty raise_stmt_rule(Parser *p); static stmt_ty function_def_rule(Parser *p); @@ -728,9 +734,9 @@ static asdl_seq *_gather_51_rule(Parser *p); static void *_tmp_53_rule(Parser *p); static void *_tmp_54_rule(Parser *p); static void *_tmp_55_rule(Parser *p); -static asdl_seq *_loop0_57_rule(Parser *p); -static asdl_seq *_gather_56_rule(Parser *p); -static void *_tmp_58_rule(Parser *p); +static void *_tmp_56_rule(Parser *p); +static asdl_seq *_loop0_58_rule(Parser *p); +static asdl_seq *_gather_57_rule(Parser *p); static asdl_seq *_loop0_60_rule(Parser *p); static asdl_seq *_gather_59_rule(Parser *p); static void *_tmp_61_rule(Parser *p); @@ -5201,7 +5207,7 @@ case_block_rule(Parser *p) Token * _literal; asdl_stmt_seq* body; void *guard; - expr_ty pattern; + pattern_ty pattern; if ( (_keyword = _PyPegen_expect_soft_keyword(p, "case")) // soft_keyword='"case"' && @@ -5297,7 +5303,7 @@ guard_rule(Parser *p) } // patterns: open_sequence_pattern | pattern -static expr_ty +static pattern_ty patterns_rule(Parser *p) { D(p->level++); @@ -5305,7 +5311,7 @@ patterns_rule(Parser *p) D(p->level--); return NULL; } - expr_ty _res = NULL; + pattern_ty _res = NULL; int _mark = p->mark; if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { p->error_indicator = 1; @@ -5322,9 +5328,9 @@ patterns_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> patterns[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "open_sequence_pattern")); - asdl_expr_seq* values; + asdl_pattern_seq* patterns; if ( - (values = (asdl_expr_seq*)open_sequence_pattern_rule(p)) // open_sequence_pattern + (patterns = (asdl_pattern_seq*)open_sequence_pattern_rule(p)) // open_sequence_pattern ) { D(fprintf(stderr, "%*c+ patterns[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "open_sequence_pattern")); @@ -5337,7 +5343,7 @@ patterns_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_Tuple ( values , Load , EXTRA ); + _res = _PyAST_MatchSequence ( patterns , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -5355,7 +5361,7 @@ patterns_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> patterns[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "pattern")); - expr_ty pattern_var; + pattern_ty pattern_var; if ( (pattern_var = pattern_rule(p)) // pattern ) @@ -5375,7 +5381,7 @@ patterns_rule(Parser *p) } // pattern: as_pattern | or_pattern -static expr_ty +static pattern_ty pattern_rule(Parser *p) { D(p->level++); @@ -5383,7 +5389,7 @@ pattern_rule(Parser *p) D(p->level--); return NULL; } - expr_ty _res = NULL; + pattern_ty _res = NULL; int _mark = p->mark; { // as_pattern if (p->error_indicator) { @@ -5391,7 +5397,7 @@ pattern_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "as_pattern")); - expr_ty as_pattern_var; + pattern_ty as_pattern_var; if ( (as_pattern_var = as_pattern_rule(p)) // as_pattern ) @@ -5410,7 +5416,7 @@ pattern_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "or_pattern")); - expr_ty or_pattern_var; + pattern_ty or_pattern_var; if ( (or_pattern_var = or_pattern_rule(p)) // or_pattern ) @@ -5429,8 +5435,8 @@ pattern_rule(Parser *p) return _res; } -// as_pattern: or_pattern 'as' capture_pattern -static expr_ty +// as_pattern: or_pattern 'as' pattern_capture_target +static pattern_ty as_pattern_rule(Parser *p) { D(p->level++); @@ -5438,7 +5444,7 @@ as_pattern_rule(Parser *p) D(p->level--); return NULL; } - expr_ty _res = NULL; + pattern_ty _res = NULL; int _mark = p->mark; if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { p->error_indicator = 1; @@ -5449,24 +5455,24 @@ as_pattern_rule(Parser *p) UNUSED(_start_lineno); // Only used by EXTRA macro int _start_col_offset = p->tokens[_mark]->col_offset; UNUSED(_start_col_offset); // Only used by EXTRA macro - { // or_pattern 'as' capture_pattern + { // or_pattern 'as' pattern_capture_target if (p->error_indicator) { D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> as_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "or_pattern 'as' capture_pattern")); + D(fprintf(stderr, "%*c> as_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "or_pattern 'as' pattern_capture_target")); Token * _keyword; - expr_ty pattern; + pattern_ty pattern; expr_ty target; if ( (pattern = or_pattern_rule(p)) // or_pattern && (_keyword = _PyPegen_expect_token(p, 520)) // token='as' && - (target = capture_pattern_rule(p)) // capture_pattern + (target = pattern_capture_target_rule(p)) // pattern_capture_target ) { - D(fprintf(stderr, "%*c+ as_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "or_pattern 'as' capture_pattern")); + D(fprintf(stderr, "%*c+ as_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "or_pattern 'as' pattern_capture_target")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { D(p->level--); @@ -5486,7 +5492,7 @@ as_pattern_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s as_pattern[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "or_pattern 'as' capture_pattern")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "or_pattern 'as' pattern_capture_target")); } _res = NULL; done: @@ -5495,7 +5501,7 @@ as_pattern_rule(Parser *p) } // or_pattern: '|'.closed_pattern+ -static expr_ty +static pattern_ty or_pattern_rule(Parser *p) { D(p->level++); @@ -5503,7 +5509,7 @@ or_pattern_rule(Parser *p) D(p->level--); return NULL; } - expr_ty _res = NULL; + pattern_ty _res = NULL; int _mark = p->mark; if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { p->error_indicator = 1; @@ -5520,9 +5526,9 @@ or_pattern_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> or_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'|'.closed_pattern+")); - asdl_expr_seq* patterns; + asdl_pattern_seq* patterns; if ( - (patterns = (asdl_expr_seq*)_gather_51_rule(p)) // '|'.closed_pattern+ + (patterns = (asdl_pattern_seq*)_gather_51_rule(p)) // '|'.closed_pattern+ ) { D(fprintf(stderr, "%*c+ or_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'|'.closed_pattern+")); @@ -5562,7 +5568,7 @@ or_pattern_rule(Parser *p) // | sequence_pattern // | mapping_pattern // | class_pattern -static expr_ty +static pattern_ty closed_pattern_rule(Parser *p) { D(p->level++); @@ -5570,7 +5576,7 @@ closed_pattern_rule(Parser *p) D(p->level--); return NULL; } - expr_ty _res = NULL; + pattern_ty _res = NULL; int _mark = p->mark; { // literal_pattern if (p->error_indicator) { @@ -5578,7 +5584,7 @@ closed_pattern_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> closed_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "literal_pattern")); - expr_ty literal_pattern_var; + pattern_ty literal_pattern_var; if ( (literal_pattern_var = literal_pattern_rule(p)) // literal_pattern ) @@ -5597,7 +5603,7 @@ closed_pattern_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> closed_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "capture_pattern")); - expr_ty capture_pattern_var; + pattern_ty capture_pattern_var; if ( (capture_pattern_var = capture_pattern_rule(p)) // capture_pattern ) @@ -5616,7 +5622,7 @@ closed_pattern_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> closed_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "wildcard_pattern")); - expr_ty wildcard_pattern_var; + pattern_ty wildcard_pattern_var; if ( (wildcard_pattern_var = wildcard_pattern_rule(p)) // wildcard_pattern ) @@ -5635,7 +5641,7 @@ closed_pattern_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> closed_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "value_pattern")); - expr_ty value_pattern_var; + pattern_ty value_pattern_var; if ( (value_pattern_var = value_pattern_rule(p)) // value_pattern ) @@ -5654,7 +5660,7 @@ closed_pattern_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> closed_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "group_pattern")); - expr_ty group_pattern_var; + pattern_ty group_pattern_var; if ( (group_pattern_var = group_pattern_rule(p)) // group_pattern ) @@ -5673,7 +5679,7 @@ closed_pattern_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> closed_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "sequence_pattern")); - expr_ty sequence_pattern_var; + pattern_ty sequence_pattern_var; if ( (sequence_pattern_var = sequence_pattern_rule(p)) // sequence_pattern ) @@ -5692,7 +5698,7 @@ closed_pattern_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> closed_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "mapping_pattern")); - expr_ty mapping_pattern_var; + pattern_ty mapping_pattern_var; if ( (mapping_pattern_var = mapping_pattern_rule(p)) // mapping_pattern ) @@ -5711,7 +5717,7 @@ closed_pattern_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> closed_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "class_pattern")); - expr_ty class_pattern_var; + pattern_ty class_pattern_var; if ( (class_pattern_var = class_pattern_rule(p)) // class_pattern ) @@ -5732,13 +5738,12 @@ closed_pattern_rule(Parser *p) // literal_pattern: // | signed_number !('+' | '-') -// | signed_number '+' NUMBER -// | signed_number '-' NUMBER +// | complex_number // | strings // | 'None' // | 'True' // | 'False' -static expr_ty +static pattern_ty literal_pattern_rule(Parser *p) { D(p->level++); @@ -5746,7 +5751,7 @@ literal_pattern_rule(Parser *p) D(p->level--); return NULL; } - expr_ty _res = NULL; + pattern_ty _res = NULL; int _mark = p->mark; if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { p->error_indicator = 1; @@ -5763,39 +5768,47 @@ literal_pattern_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "signed_number !('+' | '-')")); - expr_ty signed_number_var; + expr_ty value; if ( - (signed_number_var = signed_number_rule(p)) // signed_number + (value = signed_number_rule(p)) // signed_number && _PyPegen_lookahead(0, _tmp_53_rule, p) ) { D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "signed_number !('+' | '-')")); - _res = signed_number_var; + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + D(p->level--); + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_MatchValue ( value , EXTRA ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + D(p->level--); + return NULL; + } goto done; } p->mark = _mark; D(fprintf(stderr, "%*c%s literal_pattern[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "signed_number !('+' | '-')")); } - { // signed_number '+' NUMBER + { // complex_number if (p->error_indicator) { D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "signed_number '+' NUMBER")); - Token * _literal; - expr_ty imag; - expr_ty real; + D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "complex_number")); + expr_ty value; if ( - (real = signed_number_rule(p)) // signed_number - && - (_literal = _PyPegen_expect_token(p, 14)) // token='+' - && - (imag = _PyPegen_number_token(p)) // NUMBER + (value = complex_number_rule(p)) // complex_number ) { - D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "signed_number '+' NUMBER")); + D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "complex_number")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { D(p->level--); @@ -5805,7 +5818,7 @@ literal_pattern_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_BinOp ( real , Add , imag , EXTRA ); + _res = _PyAST_MatchValue ( value , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -5815,26 +5828,20 @@ literal_pattern_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s literal_pattern[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "signed_number '+' NUMBER")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "complex_number")); } - { // signed_number '-' NUMBER + { // strings if (p->error_indicator) { D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "signed_number '-' NUMBER")); - Token * _literal; - expr_ty imag; - expr_ty real; + D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "strings")); + expr_ty value; if ( - (real = signed_number_rule(p)) // signed_number - && - (_literal = _PyPegen_expect_token(p, 15)) // token='-' - && - (imag = _PyPegen_number_token(p)) // NUMBER + (value = strings_rule(p)) // strings ) { - D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "signed_number '-' NUMBER")); + D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "strings")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { D(p->level--); @@ -5844,7 +5851,7 @@ literal_pattern_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_BinOp ( real , Sub , imag , EXTRA ); + _res = _PyAST_MatchValue ( value , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -5854,25 +5861,196 @@ literal_pattern_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s literal_pattern[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "signed_number '-' NUMBER")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "strings")); + } + { // 'None' + if (p->error_indicator) { + D(p->level--); + return NULL; + } + D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); + Token * _keyword; + if ( + (_keyword = _PyPegen_expect_token(p, 523)) // token='None' + ) + { + D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + D(p->level--); + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_MatchConstant ( Py_None , EXTRA ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + D(p->level--); + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s literal_pattern[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'None'")); + } + { // 'True' + if (p->error_indicator) { + D(p->level--); + return NULL; + } + D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); + Token * _keyword; + if ( + (_keyword = _PyPegen_expect_token(p, 524)) // token='True' + ) + { + D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + D(p->level--); + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_MatchConstant ( Py_True , EXTRA ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + D(p->level--); + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s literal_pattern[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'True'")); + } + { // 'False' + if (p->error_indicator) { + D(p->level--); + return NULL; + } + D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); + Token * _keyword; + if ( + (_keyword = _PyPegen_expect_token(p, 525)) // token='False' + ) + { + D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + D(p->level--); + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_MatchConstant ( Py_False , EXTRA ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + D(p->level--); + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s literal_pattern[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'False'")); + } + _res = NULL; + done: + D(p->level--); + return _res; +} + +// literal_expr: +// | signed_number !('+' | '-') +// | complex_number +// | strings +// | 'None' +// | 'True' +// | 'False' +static expr_ty +literal_expr_rule(Parser *p) +{ + D(p->level++); + if (p->error_indicator) { + D(p->level--); + return NULL; + } + expr_ty _res = NULL; + int _mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + D(p->level--); + return NULL; + } + int _start_lineno = p->tokens[_mark]->lineno; + UNUSED(_start_lineno); // Only used by EXTRA macro + int _start_col_offset = p->tokens[_mark]->col_offset; + UNUSED(_start_col_offset); // Only used by EXTRA macro + { // signed_number !('+' | '-') + if (p->error_indicator) { + D(p->level--); + return NULL; + } + D(fprintf(stderr, "%*c> literal_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "signed_number !('+' | '-')")); + expr_ty signed_number_var; + if ( + (signed_number_var = signed_number_rule(p)) // signed_number + && + _PyPegen_lookahead(0, _tmp_54_rule, p) + ) + { + D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "signed_number !('+' | '-')")); + _res = signed_number_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s literal_expr[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "signed_number !('+' | '-')")); + } + { // complex_number + if (p->error_indicator) { + D(p->level--); + return NULL; + } + D(fprintf(stderr, "%*c> literal_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "complex_number")); + expr_ty complex_number_var; + if ( + (complex_number_var = complex_number_rule(p)) // complex_number + ) + { + D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "complex_number")); + _res = complex_number_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s literal_expr[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "complex_number")); } { // strings if (p->error_indicator) { D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "strings")); + D(fprintf(stderr, "%*c> literal_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "strings")); expr_ty strings_var; if ( (strings_var = strings_rule(p)) // strings ) { - D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "strings")); + D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "strings")); _res = strings_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s literal_pattern[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s literal_expr[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "strings")); } { // 'None' @@ -5880,13 +6058,13 @@ literal_pattern_rule(Parser *p) D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c> literal_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 523)) // token='None' ) { - D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { D(p->level--); @@ -5905,7 +6083,7 @@ literal_pattern_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s literal_pattern[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s literal_expr[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'None'")); } { // 'True' @@ -5913,13 +6091,13 @@ literal_pattern_rule(Parser *p) D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c> literal_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 524)) // token='True' ) { - D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { D(p->level--); @@ -5938,7 +6116,7 @@ literal_pattern_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s literal_pattern[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s literal_expr[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'True'")); } { // 'False' @@ -5946,13 +6124,13 @@ literal_pattern_rule(Parser *p) D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c> literal_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 525)) // token='False' ) { - D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { D(p->level--); @@ -5971,7 +6149,7 @@ literal_pattern_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s literal_pattern[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s literal_expr[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'False'")); } _res = NULL; @@ -5980,6 +6158,110 @@ literal_pattern_rule(Parser *p) return _res; } +// complex_number: signed_number '+' NUMBER | signed_number '-' NUMBER +static expr_ty +complex_number_rule(Parser *p) +{ + D(p->level++); + if (p->error_indicator) { + D(p->level--); + return NULL; + } + expr_ty _res = NULL; + int _mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + D(p->level--); + return NULL; + } + int _start_lineno = p->tokens[_mark]->lineno; + UNUSED(_start_lineno); // Only used by EXTRA macro + int _start_col_offset = p->tokens[_mark]->col_offset; + UNUSED(_start_col_offset); // Only used by EXTRA macro + { // signed_number '+' NUMBER + if (p->error_indicator) { + D(p->level--); + return NULL; + } + D(fprintf(stderr, "%*c> complex_number[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "signed_number '+' NUMBER")); + Token * _literal; + expr_ty imag; + expr_ty real; + if ( + (real = signed_number_rule(p)) // signed_number + && + (_literal = _PyPegen_expect_token(p, 14)) // token='+' + && + (imag = _PyPegen_number_token(p)) // NUMBER + ) + { + D(fprintf(stderr, "%*c+ complex_number[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "signed_number '+' NUMBER")); + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + D(p->level--); + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_BinOp ( real , Add , imag , EXTRA ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + D(p->level--); + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s complex_number[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "signed_number '+' NUMBER")); + } + { // signed_number '-' NUMBER + if (p->error_indicator) { + D(p->level--); + return NULL; + } + D(fprintf(stderr, "%*c> complex_number[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "signed_number '-' NUMBER")); + Token * _literal; + expr_ty imag; + expr_ty real; + if ( + (real = signed_number_rule(p)) // signed_number + && + (_literal = _PyPegen_expect_token(p, 15)) // token='-' + && + (imag = _PyPegen_number_token(p)) // NUMBER + ) + { + D(fprintf(stderr, "%*c+ complex_number[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "signed_number '-' NUMBER")); + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + D(p->level--); + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_BinOp ( real , Sub , imag , EXTRA ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + D(p->level--); + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s complex_number[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "signed_number '-' NUMBER")); + } + _res = NULL; + done: + D(p->level--); + return _res; +} + // signed_number: NUMBER | '-' NUMBER static expr_ty signed_number_rule(Parser *p) @@ -6061,9 +6343,68 @@ signed_number_rule(Parser *p) return _res; } -// capture_pattern: !"_" NAME !('.' | '(' | '=') -static expr_ty +// capture_pattern: pattern_capture_target +static pattern_ty capture_pattern_rule(Parser *p) +{ + D(p->level++); + if (p->error_indicator) { + D(p->level--); + return NULL; + } + pattern_ty _res = NULL; + int _mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + D(p->level--); + return NULL; + } + int _start_lineno = p->tokens[_mark]->lineno; + UNUSED(_start_lineno); // Only used by EXTRA macro + int _start_col_offset = p->tokens[_mark]->col_offset; + UNUSED(_start_col_offset); // Only used by EXTRA macro + { // pattern_capture_target + if (p->error_indicator) { + D(p->level--); + return NULL; + } + D(fprintf(stderr, "%*c> capture_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "pattern_capture_target")); + expr_ty target; + if ( + (target = pattern_capture_target_rule(p)) // pattern_capture_target + ) + { + D(fprintf(stderr, "%*c+ capture_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "pattern_capture_target")); + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + D(p->level--); + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_MatchAs ( NULL , target -> v . Name . id , EXTRA ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + D(p->level--); + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s capture_pattern[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "pattern_capture_target")); + } + _res = NULL; + done: + D(p->level--); + return _res; +} + +// pattern_capture_target: !"_" NAME !('.' | '(' | '=') +static expr_ty +pattern_capture_target_rule(Parser *p) { D(p->level++); if (p->error_indicator) { @@ -6077,17 +6418,17 @@ capture_pattern_rule(Parser *p) D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> capture_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "!\"_\" NAME !('.' | '(' | '=')")); + D(fprintf(stderr, "%*c> pattern_capture_target[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "!\"_\" NAME !('.' | '(' | '=')")); expr_ty name; if ( _PyPegen_lookahead_with_string(0, _PyPegen_expect_soft_keyword, p, "_") && (name = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(0, _tmp_54_rule, p) + _PyPegen_lookahead(0, _tmp_55_rule, p) ) { - D(fprintf(stderr, "%*c+ capture_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!\"_\" NAME !('.' | '(' | '=')")); + D(fprintf(stderr, "%*c+ pattern_capture_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!\"_\" NAME !('.' | '(' | '=')")); _res = _PyPegen_set_expr_context ( p , name , Store ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -6097,7 +6438,7 @@ capture_pattern_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s capture_pattern[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s pattern_capture_target[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "!\"_\" NAME !('.' | '(' | '=')")); } _res = NULL; @@ -6107,7 +6448,7 @@ capture_pattern_rule(Parser *p) } // wildcard_pattern: "_" -static expr_ty +static pattern_ty wildcard_pattern_rule(Parser *p) { D(p->level++); @@ -6115,7 +6456,7 @@ wildcard_pattern_rule(Parser *p) D(p->level--); return NULL; } - expr_ty _res = NULL; + pattern_ty _res = NULL; int _mark = p->mark; if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { p->error_indicator = 1; @@ -6147,7 +6488,7 @@ wildcard_pattern_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_Name ( CHECK ( PyObject * , _PyPegen_new_identifier ( p , "_" ) ) , Store , EXTRA ); + _res = _PyAST_MatchAlways ( EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -6166,7 +6507,7 @@ wildcard_pattern_rule(Parser *p) } // value_pattern: attr !('.' | '(' | '=') -static expr_ty +static pattern_ty value_pattern_rule(Parser *p) { D(p->level++); @@ -6174,8 +6515,17 @@ value_pattern_rule(Parser *p) D(p->level--); return NULL; } - expr_ty _res = NULL; - int _mark = p->mark; + pattern_ty _res = NULL; + int _mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + D(p->level--); + return NULL; + } + int _start_lineno = p->tokens[_mark]->lineno; + UNUSED(_start_lineno); // Only used by EXTRA macro + int _start_col_offset = p->tokens[_mark]->col_offset; + UNUSED(_start_col_offset); // Only used by EXTRA macro { // attr !('.' | '(' | '=') if (p->error_indicator) { D(p->level--); @@ -6186,11 +6536,20 @@ value_pattern_rule(Parser *p) if ( (attr = attr_rule(p)) // attr && - _PyPegen_lookahead(0, _tmp_55_rule, p) + _PyPegen_lookahead(0, _tmp_56_rule, p) ) { D(fprintf(stderr, "%*c+ value_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "attr !('.' | '(' | '=')")); - _res = attr; + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + D(p->level--); + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_MatchValue ( attr , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -6362,7 +6721,7 @@ name_or_attr_rule(Parser *p) } // group_pattern: '(' pattern ')' -static expr_ty +static pattern_ty group_pattern_rule(Parser *p) { D(p->level++); @@ -6370,7 +6729,7 @@ group_pattern_rule(Parser *p) D(p->level--); return NULL; } - expr_ty _res = NULL; + pattern_ty _res = NULL; int _mark = p->mark; { // '(' pattern ')' if (p->error_indicator) { @@ -6380,7 +6739,7 @@ group_pattern_rule(Parser *p) D(fprintf(stderr, "%*c> group_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' pattern ')'")); Token * _literal; Token * _literal_1; - expr_ty pattern; + pattern_ty pattern; if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && @@ -6409,7 +6768,7 @@ group_pattern_rule(Parser *p) } // sequence_pattern: '[' maybe_sequence_pattern? ']' | '(' open_sequence_pattern? ')' -static expr_ty +static pattern_ty sequence_pattern_rule(Parser *p) { D(p->level++); @@ -6417,7 +6776,7 @@ sequence_pattern_rule(Parser *p) D(p->level--); return NULL; } - expr_ty _res = NULL; + pattern_ty _res = NULL; int _mark = p->mark; if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { p->error_indicator = 1; @@ -6436,11 +6795,11 @@ sequence_pattern_rule(Parser *p) D(fprintf(stderr, "%*c> sequence_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'[' maybe_sequence_pattern? ']'")); Token * _literal; Token * _literal_1; - void *values; + void *patterns; if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' && - (values = maybe_sequence_pattern_rule(p), 1) // maybe_sequence_pattern? + (patterns = maybe_sequence_pattern_rule(p), 1) // maybe_sequence_pattern? && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' ) @@ -6455,7 +6814,7 @@ sequence_pattern_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_List ( values , Load , EXTRA ); + _res = _PyAST_MatchSequence ( patterns , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -6475,11 +6834,11 @@ sequence_pattern_rule(Parser *p) D(fprintf(stderr, "%*c> sequence_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' open_sequence_pattern? ')'")); Token * _literal; Token * _literal_1; - void *values; + void *patterns; if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (values = open_sequence_pattern_rule(p), 1) // open_sequence_pattern? + (patterns = open_sequence_pattern_rule(p), 1) // open_sequence_pattern? && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -6494,7 +6853,7 @@ sequence_pattern_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_Tuple ( values , Load , EXTRA ); + _res = _PyAST_MatchSequence ( patterns , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -6530,18 +6889,18 @@ open_sequence_pattern_rule(Parser *p) } D(fprintf(stderr, "%*c> open_sequence_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "maybe_star_pattern ',' maybe_sequence_pattern?")); Token * _literal; - expr_ty value; - void *values; + pattern_ty pattern; + void *patterns; if ( - (value = maybe_star_pattern_rule(p)) // maybe_star_pattern + (pattern = maybe_star_pattern_rule(p)) // maybe_star_pattern && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (values = maybe_sequence_pattern_rule(p), 1) // maybe_sequence_pattern? + (patterns = maybe_sequence_pattern_rule(p), 1) // maybe_sequence_pattern? ) { D(fprintf(stderr, "%*c+ open_sequence_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "maybe_star_pattern ',' maybe_sequence_pattern?")); - _res = _PyPegen_seq_insert_in_front ( p , value , values ); + _res = _PyPegen_seq_insert_in_front ( p , pattern , patterns ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -6578,15 +6937,15 @@ maybe_sequence_pattern_rule(Parser *p) D(fprintf(stderr, "%*c> maybe_sequence_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.maybe_star_pattern+ ','?")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - asdl_seq * values; + asdl_seq * patterns; if ( - (values = _gather_56_rule(p)) // ','.maybe_star_pattern+ + (patterns = _gather_57_rule(p)) // ','.maybe_star_pattern+ && (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? ) { D(fprintf(stderr, "%*c+ maybe_sequence_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.maybe_star_pattern+ ','?")); - _res = values; + _res = patterns; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -6605,7 +6964,7 @@ maybe_sequence_pattern_rule(Parser *p) } // maybe_star_pattern: star_pattern | pattern -static expr_ty +static pattern_ty maybe_star_pattern_rule(Parser *p) { D(p->level++); @@ -6613,7 +6972,7 @@ maybe_star_pattern_rule(Parser *p) D(p->level--); return NULL; } - expr_ty _res = NULL; + pattern_ty _res = NULL; int _mark = p->mark; { // star_pattern if (p->error_indicator) { @@ -6621,7 +6980,7 @@ maybe_star_pattern_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> maybe_star_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_pattern")); - expr_ty star_pattern_var; + pattern_ty star_pattern_var; if ( (star_pattern_var = star_pattern_rule(p)) // star_pattern ) @@ -6640,7 +6999,7 @@ maybe_star_pattern_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> maybe_star_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "pattern")); - expr_ty pattern_var; + pattern_ty pattern_var; if ( (pattern_var = pattern_rule(p)) // pattern ) @@ -6659,8 +7018,8 @@ maybe_star_pattern_rule(Parser *p) return _res; } -// star_pattern: '*' (capture_pattern | wildcard_pattern) -static expr_ty +// star_pattern: '*' pattern_capture_target | '*' wildcard_pattern +static pattern_ty star_pattern_rule(Parser *p) { D(p->level++); @@ -6668,7 +7027,7 @@ star_pattern_rule(Parser *p) D(p->level--); return NULL; } - expr_ty _res = NULL; + pattern_ty _res = NULL; int _mark = p->mark; if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { p->error_indicator = 1; @@ -6679,21 +7038,57 @@ star_pattern_rule(Parser *p) UNUSED(_start_lineno); // Only used by EXTRA macro int _start_col_offset = p->tokens[_mark]->col_offset; UNUSED(_start_col_offset); // Only used by EXTRA macro - { // '*' (capture_pattern | wildcard_pattern) + { // '*' pattern_capture_target + if (p->error_indicator) { + D(p->level--); + return NULL; + } + D(fprintf(stderr, "%*c> star_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' pattern_capture_target")); + Token * _literal; + expr_ty target; + if ( + (_literal = _PyPegen_expect_token(p, 16)) // token='*' + && + (target = pattern_capture_target_rule(p)) // pattern_capture_target + ) + { + D(fprintf(stderr, "%*c+ star_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' pattern_capture_target")); + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + D(p->level--); + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_MatchRestOfSequence ( target -> v . Name . id , EXTRA ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + D(p->level--); + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s star_pattern[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*' pattern_capture_target")); + } + { // '*' wildcard_pattern if (p->error_indicator) { D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> star_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (capture_pattern | wildcard_pattern)")); + D(fprintf(stderr, "%*c> star_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' wildcard_pattern")); Token * _literal; - void *value; + pattern_ty wildcard_pattern_var; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (value = _tmp_58_rule(p)) // capture_pattern | wildcard_pattern + (wildcard_pattern_var = wildcard_pattern_rule(p)) // wildcard_pattern ) { - D(fprintf(stderr, "%*c+ star_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (capture_pattern | wildcard_pattern)")); + D(fprintf(stderr, "%*c+ star_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' wildcard_pattern")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { D(p->level--); @@ -6703,7 +7098,7 @@ star_pattern_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_Starred ( value , Store , EXTRA ); + _res = _PyAST_MatchRestOfSequence ( NULL , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -6713,7 +7108,7 @@ star_pattern_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s star_pattern[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*' (capture_pattern | wildcard_pattern)")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*' wildcard_pattern")); } _res = NULL; done: @@ -6722,7 +7117,7 @@ star_pattern_rule(Parser *p) } // mapping_pattern: '{' items_pattern? '}' -static expr_ty +static pattern_ty mapping_pattern_rule(Parser *p) { D(p->level++); @@ -6730,7 +7125,7 @@ mapping_pattern_rule(Parser *p) D(p->level--); return NULL; } - expr_ty _res = NULL; + pattern_ty _res = NULL; int _mark = p->mark; if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { p->error_indicator = 1; @@ -6768,7 +7163,7 @@ mapping_pattern_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_Dict ( CHECK ( asdl_expr_seq * , _PyPegen_get_keys ( p , items ) ) , CHECK ( asdl_expr_seq * , _PyPegen_get_values ( p , items ) ) , EXTRA ); + _res = _PyAST_MatchMapping ( CHECK ( asdl_expr_seq * , _PyPegen_get_pattern_keys ( p , items ) ) , CHECK ( asdl_pattern_seq * , _PyPegen_get_patterns ( p , items ) ) , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -6831,8 +7226,8 @@ items_pattern_rule(Parser *p) return _res; } -// key_value_pattern: (literal_pattern | value_pattern) ':' pattern | double_star_pattern -static KeyValuePair* +// key_value_pattern: (literal_expr | attr) ':' pattern | double_star_pattern +static KeyPatternPair* key_value_pattern_rule(Parser *p) { D(p->level++); @@ -6840,27 +7235,27 @@ key_value_pattern_rule(Parser *p) D(p->level--); return NULL; } - KeyValuePair* _res = NULL; + KeyPatternPair* _res = NULL; int _mark = p->mark; - { // (literal_pattern | value_pattern) ':' pattern + { // (literal_expr | attr) ':' pattern if (p->error_indicator) { D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> key_value_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(literal_pattern | value_pattern) ':' pattern")); + D(fprintf(stderr, "%*c> key_value_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(literal_expr | attr) ':' pattern")); Token * _literal; void *key; - expr_ty value; + pattern_ty pattern; if ( - (key = _tmp_61_rule(p)) // literal_pattern | value_pattern + (key = _tmp_61_rule(p)) // literal_expr | attr && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (value = pattern_rule(p)) // pattern + (pattern = pattern_rule(p)) // pattern ) { - D(fprintf(stderr, "%*c+ key_value_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(literal_pattern | value_pattern) ':' pattern")); - _res = _PyPegen_key_value_pair ( p , key , value ); + D(fprintf(stderr, "%*c+ key_value_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(literal_expr | attr) ':' pattern")); + _res = _PyPegen_key_pattern_pair ( p , key , pattern ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -6870,7 +7265,7 @@ key_value_pattern_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s key_value_pattern[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(literal_pattern | value_pattern) ':' pattern")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(literal_expr | attr) ':' pattern")); } { // double_star_pattern if (p->error_indicator) { @@ -6878,7 +7273,7 @@ key_value_pattern_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> key_value_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_star_pattern")); - KeyValuePair* double_star_pattern_var; + KeyPatternPair* double_star_pattern_var; if ( (double_star_pattern_var = double_star_pattern_rule(p)) // double_star_pattern ) @@ -6897,8 +7292,8 @@ key_value_pattern_rule(Parser *p) return _res; } -// double_star_pattern: '**' capture_pattern -static KeyValuePair* +// double_star_pattern: '**' pattern_capture_target +static KeyPatternPair* double_star_pattern_rule(Parser *p) { D(p->level++); @@ -6906,24 +7301,42 @@ double_star_pattern_rule(Parser *p) D(p->level--); return NULL; } - KeyValuePair* _res = NULL; + KeyPatternPair* _res = NULL; int _mark = p->mark; - { // '**' capture_pattern + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + D(p->level--); + return NULL; + } + int _start_lineno = p->tokens[_mark]->lineno; + UNUSED(_start_lineno); // Only used by EXTRA macro + int _start_col_offset = p->tokens[_mark]->col_offset; + UNUSED(_start_col_offset); // Only used by EXTRA macro + { // '**' pattern_capture_target if (p->error_indicator) { D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> double_star_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**' capture_pattern")); + D(fprintf(stderr, "%*c> double_star_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**' pattern_capture_target")); Token * _literal; - expr_ty value; + expr_ty target; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' && - (value = capture_pattern_rule(p)) // capture_pattern + (target = pattern_capture_target_rule(p)) // pattern_capture_target ) { - D(fprintf(stderr, "%*c+ double_star_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' capture_pattern")); - _res = _PyPegen_key_value_pair ( p , NULL , value ); + D(fprintf(stderr, "%*c+ double_star_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' pattern_capture_target")); + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + D(p->level--); + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyPegen_key_pattern_pair ( p , NULL , _PyAST_MatchAs ( NULL , target -> v . Name . id , EXTRA ) ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -6933,7 +7346,7 @@ double_star_pattern_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s double_star_pattern[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**' capture_pattern")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**' pattern_capture_target")); } _res = NULL; done: @@ -6946,7 +7359,7 @@ double_star_pattern_rule(Parser *p) // | name_or_attr '(' positional_patterns ','? ')' // | name_or_attr '(' keyword_patterns ','? ')' // | name_or_attr '(' positional_patterns ',' keyword_patterns ','? ')' -static expr_ty +static pattern_ty class_pattern_rule(Parser *p) { D(p->level++); @@ -6954,7 +7367,7 @@ class_pattern_rule(Parser *p) D(p->level--); return NULL; } - expr_ty _res = NULL; + pattern_ty _res = NULL; int _mark = p->mark; if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { p->error_indicator = 1; @@ -6973,9 +7386,9 @@ class_pattern_rule(Parser *p) D(fprintf(stderr, "%*c> class_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "name_or_attr '(' ')'")); Token * _literal; Token * _literal_1; - expr_ty func; + expr_ty cls; if ( - (func = name_or_attr_rule(p)) // name_or_attr + (cls = name_or_attr_rule(p)) // name_or_attr && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && @@ -6992,7 +7405,7 @@ class_pattern_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_Call ( func , NULL , NULL , EXTRA ); + _res = _PyAST_MatchClass ( cls , NULL , NULL , NULL , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -7014,14 +7427,14 @@ class_pattern_rule(Parser *p) Token * _literal_1; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - asdl_expr_seq* args; - expr_ty func; + expr_ty cls; + asdl_pattern_seq* patterns; if ( - (func = name_or_attr_rule(p)) // name_or_attr + (cls = name_or_attr_rule(p)) // name_or_attr && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (args = positional_patterns_rule(p)) // positional_patterns + (patterns = positional_patterns_rule(p)) // positional_patterns && (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? && @@ -7038,7 +7451,7 @@ class_pattern_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_Call ( func , args , NULL , EXTRA ); + _res = _PyAST_MatchClass ( cls , patterns , NULL , NULL , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -7060,10 +7473,10 @@ class_pattern_rule(Parser *p) Token * _literal_1; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - expr_ty func; - asdl_keyword_seq* keywords; + expr_ty cls; + asdl_seq* keywords; if ( - (func = name_or_attr_rule(p)) // name_or_attr + (cls = name_or_attr_rule(p)) // name_or_attr && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && @@ -7084,7 +7497,7 @@ class_pattern_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_Call ( func , NULL , keywords , EXTRA ); + _res = _PyAST_MatchClass ( cls , NULL , CHECK ( asdl_identifier_seq * , _PyPegen_map_names_to_ids ( p , CHECK ( asdl_expr_seq * , _PyPegen_get_pattern_keys ( p , keywords ) ) ) ) , CHECK ( asdl_pattern_seq * , _PyPegen_get_patterns ( p , keywords ) ) , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -7107,15 +7520,15 @@ class_pattern_rule(Parser *p) Token * _literal_2; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - asdl_expr_seq* args; - expr_ty func; - asdl_keyword_seq* keywords; + expr_ty cls; + asdl_seq* keywords; + asdl_pattern_seq* patterns; if ( - (func = name_or_attr_rule(p)) // name_or_attr + (cls = name_or_attr_rule(p)) // name_or_attr && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (args = positional_patterns_rule(p)) // positional_patterns + (patterns = positional_patterns_rule(p)) // positional_patterns && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && @@ -7136,7 +7549,7 @@ class_pattern_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_Call ( func , args , keywords , EXTRA ); + _res = _PyAST_MatchClass ( cls , patterns , CHECK ( asdl_identifier_seq * , _PyPegen_map_names_to_ids ( p , CHECK ( asdl_expr_seq * , _PyPegen_get_pattern_keys ( p , keywords ) ) ) ) , CHECK ( asdl_pattern_seq * , _PyPegen_get_patterns ( p , keywords ) ) , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -7155,7 +7568,7 @@ class_pattern_rule(Parser *p) } // positional_patterns: ','.pattern+ -static asdl_expr_seq* +static asdl_pattern_seq* positional_patterns_rule(Parser *p) { D(p->level++); @@ -7163,7 +7576,7 @@ positional_patterns_rule(Parser *p) D(p->level--); return NULL; } - asdl_expr_seq* _res = NULL; + asdl_pattern_seq* _res = NULL; int _mark = p->mark; { // ','.pattern+ if (p->error_indicator) { @@ -7171,9 +7584,9 @@ positional_patterns_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> positional_patterns[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.pattern+")); - asdl_expr_seq* args; + asdl_pattern_seq* args; if ( - (args = (asdl_expr_seq*)_gather_62_rule(p)) // ','.pattern+ + (args = (asdl_pattern_seq*)_gather_62_rule(p)) // ','.pattern+ ) { D(fprintf(stderr, "%*c+ positional_patterns[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.pattern+")); @@ -7196,7 +7609,7 @@ positional_patterns_rule(Parser *p) } // keyword_patterns: ','.keyword_pattern+ -static asdl_keyword_seq* +static asdl_seq* keyword_patterns_rule(Parser *p) { D(p->level++); @@ -7204,7 +7617,7 @@ keyword_patterns_rule(Parser *p) D(p->level--); return NULL; } - asdl_keyword_seq* _res = NULL; + asdl_seq* _res = NULL; int _mark = p->mark; { // ','.keyword_pattern+ if (p->error_indicator) { @@ -7212,9 +7625,9 @@ keyword_patterns_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> keyword_patterns[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.keyword_pattern+")); - asdl_keyword_seq* keywords; + asdl_seq* keywords; if ( - (keywords = (asdl_keyword_seq*)_gather_64_rule(p)) // ','.keyword_pattern+ + (keywords = (asdl_seq*)_gather_64_rule(p)) // ','.keyword_pattern+ ) { D(fprintf(stderr, "%*c+ keyword_patterns[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.keyword_pattern+")); @@ -7237,7 +7650,7 @@ keyword_patterns_rule(Parser *p) } // keyword_pattern: NAME '=' pattern -static keyword_ty +static pattern_ty keyword_pattern_rule(Parser *p) { D(p->level++); @@ -7245,17 +7658,8 @@ keyword_pattern_rule(Parser *p) D(p->level--); return NULL; } - keyword_ty _res = NULL; + pattern_ty _res = NULL; int _mark = p->mark; - if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { - p->error_indicator = 1; - D(p->level--); - return NULL; - } - int _start_lineno = p->tokens[_mark]->lineno; - UNUSED(_start_lineno); // Only used by EXTRA macro - int _start_col_offset = p->tokens[_mark]->col_offset; - UNUSED(_start_col_offset); // Only used by EXTRA macro { // NAME '=' pattern if (p->error_indicator) { D(p->level--); @@ -7264,7 +7668,7 @@ keyword_pattern_rule(Parser *p) D(fprintf(stderr, "%*c> keyword_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME '=' pattern")); Token * _literal; expr_ty arg; - expr_ty value; + pattern_ty value; if ( (arg = _PyPegen_name_token(p)) // NAME && @@ -7274,16 +7678,7 @@ keyword_pattern_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ keyword_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '=' pattern")); - Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); - if (_token == NULL) { - D(p->level--); - return NULL; - } - int _end_lineno = _token->end_lineno; - UNUSED(_end_lineno); // Only used by EXTRA macro - int _end_col_offset = _token->end_col_offset; - UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_keyword ( arg -> v . Name . id , value , EXTRA ); + _res = _PyPegen_key_pattern_pair ( p , arg , value ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -19076,7 +19471,7 @@ invalid_case_block_rule(Parser *p) expr_ty _keyword; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - expr_ty patterns_var; + pattern_ty patterns_var; if ( (_keyword = _PyPegen_expect_soft_keyword(p, "case")) // soft_keyword='"case"' && @@ -22383,7 +22778,7 @@ _loop0_52_rule(Parser *p) } D(fprintf(stderr, "%*c> _loop0_52[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'|' closed_pattern")); Token * _literal; - expr_ty elem; + pattern_ty elem; while ( (_literal = _PyPegen_expect_token(p, 18)) // token='|' && @@ -22447,7 +22842,7 @@ _gather_51_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _gather_51[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "closed_pattern _loop0_52")); - expr_ty elem; + pattern_ty elem; asdl_seq * seq; if ( (elem = closed_pattern_rule(p)) // closed_pattern @@ -22524,9 +22919,64 @@ _tmp_53_rule(Parser *p) return _res; } -// _tmp_54: '.' | '(' | '=' +// _tmp_54: '+' | '-' static void * _tmp_54_rule(Parser *p) +{ + D(p->level++); + if (p->error_indicator) { + D(p->level--); + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // '+' + if (p->error_indicator) { + D(p->level--); + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_54[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 14)) // token='+' + ) + { + D(fprintf(stderr, "%*c+ _tmp_54[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_54[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); + } + { // '-' + if (p->error_indicator) { + D(p->level--); + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_54[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 15)) // token='-' + ) + { + D(fprintf(stderr, "%*c+ _tmp_54[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_54[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); + } + _res = NULL; + done: + D(p->level--); + return _res; +} + +// _tmp_55: '.' | '(' | '=' +static void * +_tmp_55_rule(Parser *p) { D(p->level++); if (p->error_indicator) { @@ -22540,18 +22990,18 @@ _tmp_54_rule(Parser *p) D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> _tmp_54[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_55[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_54[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_55[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_54[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_55[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '(' @@ -22559,18 +23009,18 @@ _tmp_54_rule(Parser *p) D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> _tmp_54[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c> _tmp_55[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' ) { - D(fprintf(stderr, "%*c+ _tmp_54[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c+ _tmp_55[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_54[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_55[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'('")); } { // '=' @@ -22578,18 +23028,18 @@ _tmp_54_rule(Parser *p) D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> _tmp_54[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_55[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_54[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_55[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_54[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_55[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } _res = NULL; @@ -22598,9 +23048,9 @@ _tmp_54_rule(Parser *p) return _res; } -// _tmp_55: '.' | '(' | '=' +// _tmp_56: '.' | '(' | '=' static void * -_tmp_55_rule(Parser *p) +_tmp_56_rule(Parser *p) { D(p->level++); if (p->error_indicator) { @@ -22614,18 +23064,18 @@ _tmp_55_rule(Parser *p) D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> _tmp_55[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_56[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_55[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_56[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_55[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_56[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '(' @@ -22633,18 +23083,18 @@ _tmp_55_rule(Parser *p) D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> _tmp_55[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c> _tmp_56[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' ) { - D(fprintf(stderr, "%*c+ _tmp_55[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c+ _tmp_56[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_55[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_56[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'('")); } { // '=' @@ -22652,18 +23102,18 @@ _tmp_55_rule(Parser *p) D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> _tmp_55[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_56[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_55[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_56[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_55[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_56[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } _res = NULL; @@ -22672,9 +23122,9 @@ _tmp_55_rule(Parser *p) return _res; } -// _loop0_57: ',' maybe_star_pattern +// _loop0_58: ',' maybe_star_pattern static asdl_seq * -_loop0_57_rule(Parser *p) +_loop0_58_rule(Parser *p) { D(p->level++); if (p->error_indicator) { @@ -22698,9 +23148,9 @@ _loop0_57_rule(Parser *p) D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> _loop0_57[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' maybe_star_pattern")); + D(fprintf(stderr, "%*c> _loop0_58[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' maybe_star_pattern")); Token * _literal; - expr_ty elem; + pattern_ty elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -22729,7 +23179,7 @@ _loop0_57_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_57[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_58[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' maybe_star_pattern")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -22742,14 +23192,14 @@ _loop0_57_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_57_type, _seq); + _PyPegen_insert_memo(p, _start_mark, _loop0_58_type, _seq); D(p->level--); return _seq; } -// _gather_56: maybe_star_pattern _loop0_57 +// _gather_57: maybe_star_pattern _loop0_58 static asdl_seq * -_gather_56_rule(Parser *p) +_gather_57_rule(Parser *p) { D(p->level++); if (p->error_indicator) { @@ -22758,82 +23208,27 @@ _gather_56_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // maybe_star_pattern _loop0_57 + { // maybe_star_pattern _loop0_58 if (p->error_indicator) { D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> _gather_56[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "maybe_star_pattern _loop0_57")); - expr_ty elem; + D(fprintf(stderr, "%*c> _gather_57[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "maybe_star_pattern _loop0_58")); + pattern_ty elem; asdl_seq * seq; if ( (elem = maybe_star_pattern_rule(p)) // maybe_star_pattern && - (seq = _loop0_57_rule(p)) // _loop0_57 + (seq = _loop0_58_rule(p)) // _loop0_58 ) { - D(fprintf(stderr, "%*c+ _gather_56[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "maybe_star_pattern _loop0_57")); + D(fprintf(stderr, "%*c+ _gather_57[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "maybe_star_pattern _loop0_58")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_56[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "maybe_star_pattern _loop0_57")); - } - _res = NULL; - done: - D(p->level--); - return _res; -} - -// _tmp_58: capture_pattern | wildcard_pattern -static void * -_tmp_58_rule(Parser *p) -{ - D(p->level++); - if (p->error_indicator) { - D(p->level--); - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // capture_pattern - if (p->error_indicator) { - D(p->level--); - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_58[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "capture_pattern")); - expr_ty capture_pattern_var; - if ( - (capture_pattern_var = capture_pattern_rule(p)) // capture_pattern - ) - { - D(fprintf(stderr, "%*c+ _tmp_58[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "capture_pattern")); - _res = capture_pattern_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_58[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "capture_pattern")); - } - { // wildcard_pattern - if (p->error_indicator) { - D(p->level--); - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_58[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "wildcard_pattern")); - expr_ty wildcard_pattern_var; - if ( - (wildcard_pattern_var = wildcard_pattern_rule(p)) // wildcard_pattern - ) - { - D(fprintf(stderr, "%*c+ _tmp_58[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "wildcard_pattern")); - _res = wildcard_pattern_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_58[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "wildcard_pattern")); + D(fprintf(stderr, "%*c%s _gather_57[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "maybe_star_pattern _loop0_58")); } _res = NULL; done: @@ -22869,7 +23264,7 @@ _loop0_60_rule(Parser *p) } D(fprintf(stderr, "%*c> _loop0_60[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' key_value_pattern")); Token * _literal; - KeyValuePair* elem; + KeyPatternPair* elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -22933,7 +23328,7 @@ _gather_59_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _gather_59[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "key_value_pattern _loop0_60")); - KeyValuePair* elem; + KeyPatternPair* elem; asdl_seq * seq; if ( (elem = key_value_pattern_rule(p)) // key_value_pattern @@ -22955,7 +23350,7 @@ _gather_59_rule(Parser *p) return _res; } -// _tmp_61: literal_pattern | value_pattern +// _tmp_61: literal_expr | attr static void * _tmp_61_rule(Parser *p) { @@ -22966,43 +23361,43 @@ _tmp_61_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // literal_pattern + { // literal_expr if (p->error_indicator) { D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> _tmp_61[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "literal_pattern")); - expr_ty literal_pattern_var; + D(fprintf(stderr, "%*c> _tmp_61[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "literal_expr")); + expr_ty literal_expr_var; if ( - (literal_pattern_var = literal_pattern_rule(p)) // literal_pattern + (literal_expr_var = literal_expr_rule(p)) // literal_expr ) { - D(fprintf(stderr, "%*c+ _tmp_61[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "literal_pattern")); - _res = literal_pattern_var; + D(fprintf(stderr, "%*c+ _tmp_61[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "literal_expr")); + _res = literal_expr_var; goto done; } p->mark = _mark; D(fprintf(stderr, "%*c%s _tmp_61[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "literal_pattern")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "literal_expr")); } - { // value_pattern + { // attr if (p->error_indicator) { D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> _tmp_61[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "value_pattern")); - expr_ty value_pattern_var; + D(fprintf(stderr, "%*c> _tmp_61[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "attr")); + expr_ty attr_var; if ( - (value_pattern_var = value_pattern_rule(p)) // value_pattern + (attr_var = attr_rule(p)) // attr ) { - D(fprintf(stderr, "%*c+ _tmp_61[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "value_pattern")); - _res = value_pattern_var; + D(fprintf(stderr, "%*c+ _tmp_61[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "attr")); + _res = attr_var; goto done; } p->mark = _mark; D(fprintf(stderr, "%*c%s _tmp_61[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "value_pattern")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "attr")); } _res = NULL; done: @@ -23038,7 +23433,7 @@ _loop0_63_rule(Parser *p) } D(fprintf(stderr, "%*c> _loop0_63[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' pattern")); Token * _literal; - expr_ty elem; + pattern_ty elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -23102,7 +23497,7 @@ _gather_62_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _gather_62[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "pattern _loop0_63")); - expr_ty elem; + pattern_ty elem; asdl_seq * seq; if ( (elem = pattern_rule(p)) // pattern @@ -23152,7 +23547,7 @@ _loop0_65_rule(Parser *p) } D(fprintf(stderr, "%*c> _loop0_65[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' keyword_pattern")); Token * _literal; - keyword_ty elem; + pattern_ty elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -23216,7 +23611,7 @@ _gather_64_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _gather_64[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "keyword_pattern _loop0_65")); - keyword_ty elem; + pattern_ty elem; asdl_seq * seq; if ( (elem = keyword_pattern_rule(p)) // keyword_pattern diff --git a/Parser/pegen.c b/Parser/pegen.c index eb70ea7d24fec1..e147cedba62577 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -1794,6 +1794,51 @@ _PyPegen_get_values(Parser *p, asdl_seq *seq) return new_seq; } +/* Constructs a KeyPatternPair that is used when parsing a mapping pattern */ +KeyPatternPair * +_PyPegen_key_pattern_pair(Parser *p, expr_ty key, pattern_ty pattern) +{ + KeyPatternPair *a = _PyArena_Malloc(p->arena, sizeof(KeyPatternPair)); + if (!a) { + return NULL; + } + a->key = key; + a->pattern = pattern; + return a; +} + +/* Extracts all keys from an asdl_seq* of KeyPatternPair*'s */ +asdl_expr_seq * +_PyPegen_get_pattern_keys(Parser *p, asdl_seq *seq) +{ + Py_ssize_t len = asdl_seq_LEN(seq); + asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(len, p->arena); + if (!new_seq) { + return NULL; + } + for (Py_ssize_t i = 0; i < len; i++) { + KeyPatternPair *pair = asdl_seq_GET_UNTYPED(seq, i); + asdl_seq_SET(new_seq, i, pair->key); + } + return new_seq; +} + +/* Extracts all patterns from an asdl_seq* of KeyPatternPair*'s */ +asdl_pattern_seq * +_PyPegen_get_patterns(Parser *p, asdl_seq *seq) +{ + Py_ssize_t len = asdl_seq_LEN(seq); + asdl_pattern_seq *new_seq = _Py_asdl_pattern_seq_new(len, p->arena); + if (!new_seq) { + return NULL; + } + for (Py_ssize_t i = 0; i < len; i++) { + KeyPatternPair *pair = asdl_seq_GET_UNTYPED(seq, i); + asdl_seq_SET(new_seq, i, pair->pattern); + } + return new_seq; +} + /* Constructs a NameDefaultPair */ NameDefaultPair * _PyPegen_name_default_pair(Parser *p, arg_ty arg, expr_ty value, Token *tc) diff --git a/Parser/pegen.h b/Parser/pegen.h index 9b218d4c59c4f9..e9c910a316407f 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -86,6 +86,11 @@ typedef struct { expr_ty value; } KeyValuePair; +typedef struct { + expr_ty key; + pattern_ty pattern; +} KeyPatternPair; + typedef struct { arg_ty arg; expr_ty value; @@ -248,6 +253,9 @@ expr_ty _PyPegen_set_expr_context(Parser *, expr_ty, expr_context_ty); KeyValuePair *_PyPegen_key_value_pair(Parser *, expr_ty, expr_ty); asdl_expr_seq *_PyPegen_get_keys(Parser *, asdl_seq *); asdl_expr_seq *_PyPegen_get_values(Parser *, asdl_seq *); +KeyPatternPair *_PyPegen_key_pattern_pair(Parser *, expr_ty, pattern_ty); +asdl_expr_seq *_PyPegen_get_pattern_keys(Parser *, asdl_seq *); +asdl_pattern_seq *_PyPegen_get_patterns(Parser *, asdl_seq *); NameDefaultPair *_PyPegen_name_default_pair(Parser *, arg_ty, expr_ty, Token *); SlashWithDefault *_PyPegen_slash_with_default(Parser *, asdl_arg_seq *, asdl_seq *); StarEtc *_PyPegen_star_etc(Parser *, arg_ty, asdl_seq *, arg_ty); diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 779ee3e48946d5..bf435432900f6d 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -105,8 +105,15 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(state->Lt_type); Py_CLEAR(state->MatMult_singleton); Py_CLEAR(state->MatMult_type); + Py_CLEAR(state->MatchAlways_type); Py_CLEAR(state->MatchAs_type); + Py_CLEAR(state->MatchClass_type); + Py_CLEAR(state->MatchConstant_type); + Py_CLEAR(state->MatchMapping_type); Py_CLEAR(state->MatchOr_type); + Py_CLEAR(state->MatchRestOfSequence_type); + Py_CLEAR(state->MatchSequence_type); + Py_CLEAR(state->MatchValue_type); Py_CLEAR(state->Match_type); Py_CLEAR(state->Mod_singleton); Py_CLEAR(state->Mod_type); @@ -173,6 +180,7 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(state->boolop_type); Py_CLEAR(state->cases); Py_CLEAR(state->cause); + Py_CLEAR(state->cls); Py_CLEAR(state->cmpop_type); Py_CLEAR(state->col_offset); Py_CLEAR(state->comparators); @@ -190,6 +198,8 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(state->excepthandler_type); Py_CLEAR(state->expr_context_type); Py_CLEAR(state->expr_type); + Py_CLEAR(state->extra_attrs); + Py_CLEAR(state->extra_patterns); Py_CLEAR(state->finalbody); Py_CLEAR(state->format_spec); Py_CLEAR(state->func); @@ -226,6 +236,7 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(state->optional_vars); Py_CLEAR(state->orelse); Py_CLEAR(state->pattern); + Py_CLEAR(state->pattern_type); Py_CLEAR(state->patterns); Py_CLEAR(state->posonlyargs); Py_CLEAR(state->returns); @@ -276,6 +287,7 @@ static int init_identifiers(struct ast_state *state) if ((state->body = PyUnicode_InternFromString("body")) == NULL) return 0; if ((state->cases = PyUnicode_InternFromString("cases")) == NULL) return 0; if ((state->cause = PyUnicode_InternFromString("cause")) == NULL) return 0; + if ((state->cls = PyUnicode_InternFromString("cls")) == NULL) return 0; if ((state->col_offset = PyUnicode_InternFromString("col_offset")) == NULL) return 0; if ((state->comparators = PyUnicode_InternFromString("comparators")) == NULL) return 0; if ((state->context_expr = PyUnicode_InternFromString("context_expr")) == NULL) return 0; @@ -288,6 +300,8 @@ static int init_identifiers(struct ast_state *state) if ((state->end_col_offset = PyUnicode_InternFromString("end_col_offset")) == NULL) return 0; if ((state->end_lineno = PyUnicode_InternFromString("end_lineno")) == NULL) return 0; if ((state->exc = PyUnicode_InternFromString("exc")) == NULL) return 0; + if ((state->extra_attrs = PyUnicode_InternFromString("extra_attrs")) == NULL) return 0; + if ((state->extra_patterns = PyUnicode_InternFromString("extra_patterns")) == NULL) return 0; if ((state->finalbody = PyUnicode_InternFromString("finalbody")) == NULL) return 0; if ((state->format_spec = PyUnicode_InternFromString("format_spec")) == NULL) return 0; if ((state->func = PyUnicode_InternFromString("func")) == NULL) return 0; @@ -353,6 +367,7 @@ GENERATE_ASDL_SEQ_CONSTRUCTOR(keyword, keyword_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(alias, alias_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(withitem, withitem_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(match_case, match_case_ty) +GENERATE_ASDL_SEQ_CONSTRUCTOR(pattern, pattern_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(type_ignore, type_ignore_ty) static PyObject* ast2obj_mod(struct ast_state *state, void*); @@ -610,13 +625,6 @@ static const char * const Slice_fields[]={ "upper", "step", }; -static const char * const MatchAs_fields[]={ - "pattern", - "name", -}; -static const char * const MatchOr_fields[]={ - "patterns", -}; static PyObject* ast2obj_expr_context(struct ast_state *state, expr_context_ty); static PyObject* ast2obj_boolop(struct ast_state *state, boolop_ty); static PyObject* ast2obj_operator(struct ast_state *state, operator_ty); @@ -696,6 +704,42 @@ static const char * const match_case_fields[]={ "guard", "body", }; +static const char * const pattern_attributes[] = { + "lineno", + "col_offset", + "end_lineno", + "end_col_offset", +}; +static PyObject* ast2obj_pattern(struct ast_state *state, void*); +static const char * const MatchValue_fields[]={ + "value", +}; +static const char * const MatchConstant_fields[]={ + "value", +}; +static const char * const MatchSequence_fields[]={ + "patterns", +}; +static const char * const MatchMapping_fields[]={ + "keys", + "patterns", +}; +static const char * const MatchClass_fields[]={ + "cls", + "patterns", + "extra_attrs", + "extra_patterns", +}; +static const char * const MatchRestOfSequence_fields[]={ + "target", +}; +static const char * const MatchAs_fields[]={ + "pattern", + "target", +}; +static const char * const MatchOr_fields[]={ + "patterns", +}; static PyObject* ast2obj_type_ignore(struct ast_state *state, void*); static const char * const TypeIgnore_fields[]={ "lineno", @@ -1275,9 +1319,7 @@ init_types(struct ast_state *state) " | Name(identifier id, expr_context ctx)\n" " | List(expr* elts, expr_context ctx)\n" " | Tuple(expr* elts, expr_context ctx)\n" - " | Slice(expr? lower, expr? upper, expr? step)\n" - " | MatchAs(expr pattern, identifier name)\n" - " | MatchOr(expr* patterns)"); + " | Slice(expr? lower, expr? upper, expr? step)"); if (!state->expr_type) return 0; if (!add_attributes(state, state->expr_type, expr_attributes, 4)) return 0; if (PyObject_SetAttr(state->expr_type, state->end_lineno, Py_None) == -1) @@ -1410,14 +1452,6 @@ init_types(struct ast_state *state) return 0; if (PyObject_SetAttr(state->Slice_type, state->step, Py_None) == -1) return 0; - state->MatchAs_type = make_type(state, "MatchAs", state->expr_type, - MatchAs_fields, 2, - "MatchAs(expr pattern, identifier name)"); - if (!state->MatchAs_type) return 0; - state->MatchOr_type = make_type(state, "MatchOr", state->expr_type, - MatchOr_fields, 1, - "MatchOr(expr* patterns)"); - if (!state->MatchOr_type) return 0; state->expr_context_type = make_type(state, "expr_context", state->AST_type, NULL, 0, "expr_context = Load | Store | Del"); @@ -1732,11 +1766,76 @@ init_types(struct ast_state *state) return 0; state->match_case_type = make_type(state, "match_case", state->AST_type, match_case_fields, 3, - "match_case(expr pattern, expr? guard, stmt* body)"); + "match_case(pattern pattern, expr? guard, stmt* body)"); if (!state->match_case_type) return 0; if (!add_attributes(state, state->match_case_type, NULL, 0)) return 0; if (PyObject_SetAttr(state->match_case_type, state->guard, Py_None) == -1) return 0; + state->pattern_type = make_type(state, "pattern", state->AST_type, NULL, 0, + "pattern = MatchAlways\n" + " | MatchValue(expr value)\n" + " | MatchConstant(constant value)\n" + " | MatchSequence(pattern* patterns)\n" + " | MatchMapping(expr* keys, pattern* patterns)\n" + " | MatchClass(expr cls, pattern* patterns, identifier* extra_attrs, pattern* extra_patterns)\n" + " | MatchRestOfSequence(identifier? target)\n" + " | MatchAs(pattern? pattern, identifier target)\n" + " | MatchOr(pattern* patterns)"); + if (!state->pattern_type) return 0; + if (!add_attributes(state, state->pattern_type, pattern_attributes, 4)) + return 0; + if (PyObject_SetAttr(state->pattern_type, state->end_lineno, Py_None) == -1) + return 0; + if (PyObject_SetAttr(state->pattern_type, state->end_col_offset, Py_None) + == -1) + return 0; + state->MatchAlways_type = make_type(state, "MatchAlways", + state->pattern_type, NULL, 0, + "MatchAlways"); + if (!state->MatchAlways_type) return 0; + state->MatchValue_type = make_type(state, "MatchValue", + state->pattern_type, MatchValue_fields, + 1, + "MatchValue(expr value)"); + if (!state->MatchValue_type) return 0; + state->MatchConstant_type = make_type(state, "MatchConstant", + state->pattern_type, + MatchConstant_fields, 1, + "MatchConstant(constant value)"); + if (!state->MatchConstant_type) return 0; + state->MatchSequence_type = make_type(state, "MatchSequence", + state->pattern_type, + MatchSequence_fields, 1, + "MatchSequence(pattern* patterns)"); + if (!state->MatchSequence_type) return 0; + state->MatchMapping_type = make_type(state, "MatchMapping", + state->pattern_type, + MatchMapping_fields, 2, + "MatchMapping(expr* keys, pattern* patterns)"); + if (!state->MatchMapping_type) return 0; + state->MatchClass_type = make_type(state, "MatchClass", + state->pattern_type, MatchClass_fields, + 4, + "MatchClass(expr cls, pattern* patterns, identifier* extra_attrs, pattern* extra_patterns)"); + if (!state->MatchClass_type) return 0; + state->MatchRestOfSequence_type = make_type(state, "MatchRestOfSequence", + state->pattern_type, + MatchRestOfSequence_fields, 1, + "MatchRestOfSequence(identifier? target)"); + if (!state->MatchRestOfSequence_type) return 0; + if (PyObject_SetAttr(state->MatchRestOfSequence_type, state->target, + Py_None) == -1) + return 0; + state->MatchAs_type = make_type(state, "MatchAs", state->pattern_type, + MatchAs_fields, 2, + "MatchAs(pattern? pattern, identifier target)"); + if (!state->MatchAs_type) return 0; + if (PyObject_SetAttr(state->MatchAs_type, state->pattern, Py_None) == -1) + return 0; + state->MatchOr_type = make_type(state, "MatchOr", state->pattern_type, + MatchOr_fields, 1, + "MatchOr(pattern* patterns)"); + if (!state->MatchOr_type) return 0; state->type_ignore_type = make_type(state, "type_ignore", state->AST_type, NULL, 0, "type_ignore = TypeIgnore(int lineno, string tag)"); @@ -1784,6 +1883,8 @@ static int obj2ast_withitem(struct ast_state *state, PyObject* obj, withitem_ty* out, PyArena* arena); static int obj2ast_match_case(struct ast_state *state, PyObject* obj, match_case_ty* out, PyArena* arena); +static int obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* + out, PyArena* arena); static int obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* out, PyArena* arena); @@ -3127,51 +3228,6 @@ _PyAST_Slice(expr_ty lower, expr_ty upper, expr_ty step, int lineno, int return p; } -expr_ty -_PyAST_MatchAs(expr_ty pattern, identifier name, int lineno, int col_offset, - int end_lineno, int end_col_offset, PyArena *arena) -{ - expr_ty p; - if (!pattern) { - PyErr_SetString(PyExc_ValueError, - "field 'pattern' is required for MatchAs"); - return NULL; - } - if (!name) { - PyErr_SetString(PyExc_ValueError, - "field 'name' is required for MatchAs"); - return NULL; - } - p = (expr_ty)_PyArena_Malloc(arena, sizeof(*p)); - if (!p) - return NULL; - p->kind = MatchAs_kind; - p->v.MatchAs.pattern = pattern; - p->v.MatchAs.name = name; - p->lineno = lineno; - p->col_offset = col_offset; - p->end_lineno = end_lineno; - p->end_col_offset = end_col_offset; - return p; -} - -expr_ty -_PyAST_MatchOr(asdl_expr_seq * patterns, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena) -{ - expr_ty p; - p = (expr_ty)_PyArena_Malloc(arena, sizeof(*p)); - if (!p) - return NULL; - p->kind = MatchOr_kind; - p->v.MatchOr.patterns = patterns; - p->lineno = lineno; - p->col_offset = col_offset; - p->end_lineno = end_lineno; - p->end_col_offset = end_col_offset; - return p; -} - comprehension_ty _PyAST_comprehension(expr_ty target, expr_ty iter, asdl_expr_seq * ifs, int is_async, PyArena *arena) @@ -3322,8 +3378,8 @@ _PyAST_withitem(expr_ty context_expr, expr_ty optional_vars, PyArena *arena) } match_case_ty -_PyAST_match_case(expr_ty pattern, expr_ty guard, asdl_stmt_seq * body, PyArena - *arena) +_PyAST_match_case(pattern_ty pattern, expr_ty guard, asdl_stmt_seq * body, + PyArena *arena) { match_case_ty p; if (!pattern) { @@ -3340,6 +3396,186 @@ _PyAST_match_case(expr_ty pattern, expr_ty guard, asdl_stmt_seq * body, PyArena return p; } +pattern_ty +_PyAST_MatchAlways(int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena) +{ + pattern_ty p; + p = (pattern_ty)_PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = MatchAlways_kind; + p->lineno = lineno; + p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; + return p; +} + +pattern_ty +_PyAST_MatchValue(expr_ty value, int lineno, int col_offset, int end_lineno, + int end_col_offset, PyArena *arena) +{ + pattern_ty p; + if (!value) { + PyErr_SetString(PyExc_ValueError, + "field 'value' is required for MatchValue"); + return NULL; + } + p = (pattern_ty)_PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = MatchValue_kind; + p->v.MatchValue.value = value; + p->lineno = lineno; + p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; + return p; +} + +pattern_ty +_PyAST_MatchConstant(constant value, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) +{ + pattern_ty p; + if (!value) { + PyErr_SetString(PyExc_ValueError, + "field 'value' is required for MatchConstant"); + return NULL; + } + p = (pattern_ty)_PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = MatchConstant_kind; + p->v.MatchConstant.value = value; + p->lineno = lineno; + p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; + return p; +} + +pattern_ty +_PyAST_MatchSequence(asdl_pattern_seq * patterns, int lineno, int col_offset, + int end_lineno, int end_col_offset, PyArena *arena) +{ + pattern_ty p; + p = (pattern_ty)_PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = MatchSequence_kind; + p->v.MatchSequence.patterns = patterns; + p->lineno = lineno; + p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; + return p; +} + +pattern_ty +_PyAST_MatchMapping(asdl_expr_seq * keys, asdl_pattern_seq * patterns, int + lineno, int col_offset, int end_lineno, int end_col_offset, + PyArena *arena) +{ + pattern_ty p; + p = (pattern_ty)_PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = MatchMapping_kind; + p->v.MatchMapping.keys = keys; + p->v.MatchMapping.patterns = patterns; + p->lineno = lineno; + p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; + return p; +} + +pattern_ty +_PyAST_MatchClass(expr_ty cls, asdl_pattern_seq * patterns, asdl_identifier_seq + * extra_attrs, asdl_pattern_seq * extra_patterns, int lineno, + int col_offset, int end_lineno, int end_col_offset, PyArena + *arena) +{ + pattern_ty p; + if (!cls) { + PyErr_SetString(PyExc_ValueError, + "field 'cls' is required for MatchClass"); + return NULL; + } + p = (pattern_ty)_PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = MatchClass_kind; + p->v.MatchClass.cls = cls; + p->v.MatchClass.patterns = patterns; + p->v.MatchClass.extra_attrs = extra_attrs; + p->v.MatchClass.extra_patterns = extra_patterns; + p->lineno = lineno; + p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; + return p; +} + +pattern_ty +_PyAST_MatchRestOfSequence(identifier target, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) +{ + pattern_ty p; + p = (pattern_ty)_PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = MatchRestOfSequence_kind; + p->v.MatchRestOfSequence.target = target; + p->lineno = lineno; + p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; + return p; +} + +pattern_ty +_PyAST_MatchAs(pattern_ty pattern, identifier target, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena *arena) +{ + pattern_ty p; + if (!target) { + PyErr_SetString(PyExc_ValueError, + "field 'target' is required for MatchAs"); + return NULL; + } + p = (pattern_ty)_PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = MatchAs_kind; + p->v.MatchAs.pattern = pattern; + p->v.MatchAs.target = target; + p->lineno = lineno; + p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; + return p; +} + +pattern_ty +_PyAST_MatchOr(asdl_pattern_seq * patterns, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) +{ + pattern_ty p; + p = (pattern_ty)_PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = MatchOr_kind; + p->v.MatchOr.patterns = patterns; + p->lineno = lineno; + p->col_offset = col_offset; + p->end_lineno = end_lineno; + p->end_col_offset = end_col_offset; + return p; +} + type_ignore_ty _PyAST_TypeIgnore(int lineno, string tag, PyArena *arena) { @@ -4410,32 +4646,6 @@ ast2obj_expr(struct ast_state *state, void* _o) goto failed; Py_DECREF(value); break; - case MatchAs_kind: - tp = (PyTypeObject *)state->MatchAs_type; - result = PyType_GenericNew(tp, NULL, NULL); - if (!result) goto failed; - value = ast2obj_expr(state, o->v.MatchAs.pattern); - if (!value) goto failed; - if (PyObject_SetAttr(result, state->pattern, value) == -1) - goto failed; - Py_DECREF(value); - value = ast2obj_identifier(state, o->v.MatchAs.name); - if (!value) goto failed; - if (PyObject_SetAttr(result, state->name, value) == -1) - goto failed; - Py_DECREF(value); - break; - case MatchOr_kind: - tp = (PyTypeObject *)state->MatchOr_type; - result = PyType_GenericNew(tp, NULL, NULL); - if (!result) goto failed; - value = ast2obj_list(state, (asdl_seq*)o->v.MatchOr.patterns, - ast2obj_expr); - if (!value) goto failed; - if (PyObject_SetAttr(result, state->patterns, value) == -1) - goto failed; - Py_DECREF(value); - break; } value = ast2obj_int(state, o->lineno); if (!value) goto failed; @@ -4935,7 +5145,7 @@ ast2obj_match_case(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->match_case_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) return NULL; - value = ast2obj_expr(state, o->pattern); + value = ast2obj_pattern(state, o->pattern); if (!value) goto failed; if (PyObject_SetAttr(result, state->pattern, value) == -1) goto failed; @@ -4958,45 +5168,200 @@ ast2obj_match_case(struct ast_state *state, void* _o) } PyObject* -ast2obj_type_ignore(struct ast_state *state, void* _o) +ast2obj_pattern(struct ast_state *state, void* _o) { - type_ignore_ty o = (type_ignore_ty)_o; + pattern_ty o = (pattern_ty)_o; PyObject *result = NULL, *value = NULL; PyTypeObject *tp; if (!o) { Py_RETURN_NONE; } switch (o->kind) { - case TypeIgnore_kind: - tp = (PyTypeObject *)state->TypeIgnore_type; + case MatchAlways_kind: + tp = (PyTypeObject *)state->MatchAlways_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_int(state, o->v.TypeIgnore.lineno); + break; + case MatchValue_kind: + tp = (PyTypeObject *)state->MatchValue_type; + result = PyType_GenericNew(tp, NULL, NULL); + if (!result) goto failed; + value = ast2obj_expr(state, o->v.MatchValue.value); if (!value) goto failed; - if (PyObject_SetAttr(result, state->lineno, value) == -1) + if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_string(state, o->v.TypeIgnore.tag); + break; + case MatchConstant_kind: + tp = (PyTypeObject *)state->MatchConstant_type; + result = PyType_GenericNew(tp, NULL, NULL); + if (!result) goto failed; + value = ast2obj_constant(state, o->v.MatchConstant.value); if (!value) goto failed; - if (PyObject_SetAttr(result, state->tag, value) == -1) + if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; Py_DECREF(value); break; - } - return result; -failed: - Py_XDECREF(value); - Py_XDECREF(result); - return NULL; -} - - -int -obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) -{ - int isinstance; - - PyObject *tmp = NULL; + case MatchSequence_kind: + tp = (PyTypeObject *)state->MatchSequence_type; + result = PyType_GenericNew(tp, NULL, NULL); + if (!result) goto failed; + value = ast2obj_list(state, (asdl_seq*)o->v.MatchSequence.patterns, + ast2obj_pattern); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->patterns, value) == -1) + goto failed; + Py_DECREF(value); + break; + case MatchMapping_kind: + tp = (PyTypeObject *)state->MatchMapping_type; + result = PyType_GenericNew(tp, NULL, NULL); + if (!result) goto failed; + value = ast2obj_list(state, (asdl_seq*)o->v.MatchMapping.keys, + ast2obj_expr); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->keys, value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_list(state, (asdl_seq*)o->v.MatchMapping.patterns, + ast2obj_pattern); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->patterns, value) == -1) + goto failed; + Py_DECREF(value); + break; + case MatchClass_kind: + tp = (PyTypeObject *)state->MatchClass_type; + result = PyType_GenericNew(tp, NULL, NULL); + if (!result) goto failed; + value = ast2obj_expr(state, o->v.MatchClass.cls); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->cls, value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_list(state, (asdl_seq*)o->v.MatchClass.patterns, + ast2obj_pattern); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->patterns, value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_list(state, (asdl_seq*)o->v.MatchClass.extra_attrs, + ast2obj_identifier); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->extra_attrs, value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_list(state, (asdl_seq*)o->v.MatchClass.extra_patterns, + ast2obj_pattern); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->extra_patterns, value) == -1) + goto failed; + Py_DECREF(value); + break; + case MatchRestOfSequence_kind: + tp = (PyTypeObject *)state->MatchRestOfSequence_type; + result = PyType_GenericNew(tp, NULL, NULL); + if (!result) goto failed; + value = ast2obj_identifier(state, o->v.MatchRestOfSequence.target); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->target, value) == -1) + goto failed; + Py_DECREF(value); + break; + case MatchAs_kind: + tp = (PyTypeObject *)state->MatchAs_type; + result = PyType_GenericNew(tp, NULL, NULL); + if (!result) goto failed; + value = ast2obj_pattern(state, o->v.MatchAs.pattern); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->pattern, value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_identifier(state, o->v.MatchAs.target); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->target, value) == -1) + goto failed; + Py_DECREF(value); + break; + case MatchOr_kind: + tp = (PyTypeObject *)state->MatchOr_type; + result = PyType_GenericNew(tp, NULL, NULL); + if (!result) goto failed; + value = ast2obj_list(state, (asdl_seq*)o->v.MatchOr.patterns, + ast2obj_pattern); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->patterns, value) == -1) + goto failed; + Py_DECREF(value); + break; + } + value = ast2obj_int(state, o->lineno); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->lineno, value) < 0) + goto failed; + Py_DECREF(value); + value = ast2obj_int(state, o->col_offset); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->col_offset, value) < 0) + goto failed; + Py_DECREF(value); + value = ast2obj_int(state, o->end_lineno); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->end_lineno, value) < 0) + goto failed; + Py_DECREF(value); + value = ast2obj_int(state, o->end_col_offset); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->end_col_offset, value) < 0) + goto failed; + Py_DECREF(value); + return result; +failed: + Py_XDECREF(value); + Py_XDECREF(result); + return NULL; +} + +PyObject* +ast2obj_type_ignore(struct ast_state *state, void* _o) +{ + type_ignore_ty o = (type_ignore_ty)_o; + PyObject *result = NULL, *value = NULL; + PyTypeObject *tp; + if (!o) { + Py_RETURN_NONE; + } + switch (o->kind) { + case TypeIgnore_kind: + tp = (PyTypeObject *)state->TypeIgnore_type; + result = PyType_GenericNew(tp, NULL, NULL); + if (!result) goto failed; + value = ast2obj_int(state, o->v.TypeIgnore.lineno); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->lineno, value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_string(state, o->v.TypeIgnore.tag); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->tag, value) == -1) + goto failed; + Py_DECREF(value); + break; + } + return result; +failed: + Py_XDECREF(value); + Py_XDECREF(result); + return NULL; +} + + +int +obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) +{ + int isinstance; + + PyObject *tmp = NULL; PyObject *tp; if (obj == Py_None) { @@ -8689,92 +9054,6 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (*out == NULL) goto failed; return 0; } - tp = state->MatchAs_type; - isinstance = PyObject_IsInstance(obj, tp); - if (isinstance == -1) { - return 1; - } - if (isinstance) { - expr_ty pattern; - identifier name; - - if (_PyObject_LookupAttr(obj, state->pattern, &tmp) < 0) { - return 1; - } - if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"pattern\" missing from MatchAs"); - return 1; - } - else { - int res; - res = obj2ast_expr(state, tmp, &pattern, arena); - if (res != 0) goto failed; - Py_CLEAR(tmp); - } - if (_PyObject_LookupAttr(obj, state->name, &tmp) < 0) { - return 1; - } - if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from MatchAs"); - return 1; - } - else { - int res; - res = obj2ast_identifier(state, tmp, &name, arena); - if (res != 0) goto failed; - Py_CLEAR(tmp); - } - *out = _PyAST_MatchAs(pattern, name, lineno, col_offset, end_lineno, - end_col_offset, arena); - if (*out == NULL) goto failed; - return 0; - } - tp = state->MatchOr_type; - isinstance = PyObject_IsInstance(obj, tp); - if (isinstance == -1) { - return 1; - } - if (isinstance) { - asdl_expr_seq* patterns; - - if (_PyObject_LookupAttr(obj, state->patterns, &tmp) < 0) { - return 1; - } - if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"patterns\" missing from MatchOr"); - return 1; - } - else { - int res; - Py_ssize_t len; - Py_ssize_t i; - if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "MatchOr field \"patterns\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); - goto failed; - } - len = PyList_GET_SIZE(tmp); - patterns = _Py_asdl_expr_seq_new(len, arena); - if (patterns == NULL) goto failed; - for (i = 0; i < len; i++) { - expr_ty val; - PyObject *tmp2 = PyList_GET_ITEM(tmp, i); - Py_INCREF(tmp2); - res = obj2ast_expr(state, tmp2, &val, arena); - Py_DECREF(tmp2); - if (res != 0) goto failed; - if (len != PyList_GET_SIZE(tmp)) { - PyErr_SetString(PyExc_RuntimeError, "MatchOr field \"patterns\" changed size during iteration"); - goto failed; - } - asdl_seq_SET(patterns, i, val); - } - Py_CLEAR(tmp); - } - *out = _PyAST_MatchOr(patterns, lineno, col_offset, end_lineno, - end_col_offset, arena); - if (*out == NULL) goto failed; - return 0; - } PyErr_Format(PyExc_TypeError, "expected some sort of expr, but got %R", obj); failed: @@ -9897,7 +10176,7 @@ obj2ast_match_case(struct ast_state *state, PyObject* obj, match_case_ty* out, PyArena* arena) { PyObject* tmp = NULL; - expr_ty pattern; + pattern_ty pattern; expr_ty guard; asdl_stmt_seq* body; @@ -9910,7 +10189,7 @@ obj2ast_match_case(struct ast_state *state, PyObject* obj, match_case_ty* out, } else { int res; - res = obj2ast_expr(state, tmp, &pattern, arena); + res = obj2ast_pattern(state, tmp, &pattern, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } @@ -9968,55 +10247,562 @@ obj2ast_match_case(struct ast_state *state, PyObject* obj, match_case_ty* out, } int -obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* - out, PyArena* arena) +obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, + PyArena* arena) { int isinstance; PyObject *tmp = NULL; PyObject *tp; + int lineno; + int col_offset; + int end_lineno; + int end_col_offset; if (obj == Py_None) { *out = NULL; return 0; } - tp = state->TypeIgnore_type; - isinstance = PyObject_IsInstance(obj, tp); - if (isinstance == -1) { + if (_PyObject_LookupAttr(obj, state->lineno, &tmp) < 0) { return 1; } - if (isinstance) { - int lineno; - string tag; - - if (_PyObject_LookupAttr(obj, state->lineno, &tmp) < 0) { - return 1; - } - if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from TypeIgnore"); - return 1; - } - else { - int res; - res = obj2ast_int(state, tmp, &lineno, arena); - if (res != 0) goto failed; - Py_CLEAR(tmp); - } - if (_PyObject_LookupAttr(obj, state->tag, &tmp) < 0) { - return 1; - } - if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"tag\" missing from TypeIgnore"); - return 1; - } - else { - int res; - res = obj2ast_string(state, tmp, &tag, arena); - if (res != 0) goto failed; - Py_CLEAR(tmp); - } - *out = _PyAST_TypeIgnore(lineno, tag, arena); - if (*out == NULL) goto failed; + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from pattern"); + return 1; + } + else { + int res; + res = obj2ast_int(state, tmp, &lineno, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + if (_PyObject_LookupAttr(obj, state->col_offset, &tmp) < 0) { + return 1; + } + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from pattern"); + return 1; + } + else { + int res; + res = obj2ast_int(state, tmp, &col_offset, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + if (_PyObject_LookupAttr(obj, state->end_lineno, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + end_lineno = 0; + } + else { + int res; + res = obj2ast_int(state, tmp, &end_lineno, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + if (_PyObject_LookupAttr(obj, state->end_col_offset, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + end_col_offset = 0; + } + else { + int res; + res = obj2ast_int(state, tmp, &end_col_offset, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + tp = state->MatchAlways_type; + isinstance = PyObject_IsInstance(obj, tp); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + + *out = _PyAST_MatchAlways(lineno, col_offset, end_lineno, + end_col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } + tp = state->MatchValue_type; + isinstance = PyObject_IsInstance(obj, tp); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + expr_ty value; + + if (_PyObject_LookupAttr(obj, state->value, &tmp) < 0) { + return 1; + } + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from MatchValue"); + return 1; + } + else { + int res; + res = obj2ast_expr(state, tmp, &value, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = _PyAST_MatchValue(value, lineno, col_offset, end_lineno, + end_col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } + tp = state->MatchConstant_type; + isinstance = PyObject_IsInstance(obj, tp); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + constant value; + + if (_PyObject_LookupAttr(obj, state->value, &tmp) < 0) { + return 1; + } + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from MatchConstant"); + return 1; + } + else { + int res; + res = obj2ast_constant(state, tmp, &value, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = _PyAST_MatchConstant(value, lineno, col_offset, end_lineno, + end_col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } + tp = state->MatchSequence_type; + isinstance = PyObject_IsInstance(obj, tp); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + asdl_pattern_seq* patterns; + + if (_PyObject_LookupAttr(obj, state->patterns, &tmp) < 0) { + return 1; + } + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"patterns\" missing from MatchSequence"); + return 1; + } + else { + int res; + Py_ssize_t len; + Py_ssize_t i; + if (!PyList_Check(tmp)) { + PyErr_Format(PyExc_TypeError, "MatchSequence field \"patterns\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); + goto failed; + } + len = PyList_GET_SIZE(tmp); + patterns = _Py_asdl_pattern_seq_new(len, arena); + if (patterns == NULL) goto failed; + for (i = 0; i < len; i++) { + pattern_ty val; + PyObject *tmp2 = PyList_GET_ITEM(tmp, i); + Py_INCREF(tmp2); + res = obj2ast_pattern(state, tmp2, &val, arena); + Py_DECREF(tmp2); + if (res != 0) goto failed; + if (len != PyList_GET_SIZE(tmp)) { + PyErr_SetString(PyExc_RuntimeError, "MatchSequence field \"patterns\" changed size during iteration"); + goto failed; + } + asdl_seq_SET(patterns, i, val); + } + Py_CLEAR(tmp); + } + *out = _PyAST_MatchSequence(patterns, lineno, col_offset, end_lineno, + end_col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } + tp = state->MatchMapping_type; + isinstance = PyObject_IsInstance(obj, tp); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + asdl_expr_seq* keys; + asdl_pattern_seq* patterns; + + if (_PyObject_LookupAttr(obj, state->keys, &tmp) < 0) { + return 1; + } + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"keys\" missing from MatchMapping"); + return 1; + } + else { + int res; + Py_ssize_t len; + Py_ssize_t i; + if (!PyList_Check(tmp)) { + PyErr_Format(PyExc_TypeError, "MatchMapping field \"keys\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); + goto failed; + } + len = PyList_GET_SIZE(tmp); + keys = _Py_asdl_expr_seq_new(len, arena); + if (keys == NULL) goto failed; + for (i = 0; i < len; i++) { + expr_ty val; + PyObject *tmp2 = PyList_GET_ITEM(tmp, i); + Py_INCREF(tmp2); + res = obj2ast_expr(state, tmp2, &val, arena); + Py_DECREF(tmp2); + if (res != 0) goto failed; + if (len != PyList_GET_SIZE(tmp)) { + PyErr_SetString(PyExc_RuntimeError, "MatchMapping field \"keys\" changed size during iteration"); + goto failed; + } + asdl_seq_SET(keys, i, val); + } + Py_CLEAR(tmp); + } + if (_PyObject_LookupAttr(obj, state->patterns, &tmp) < 0) { + return 1; + } + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"patterns\" missing from MatchMapping"); + return 1; + } + else { + int res; + Py_ssize_t len; + Py_ssize_t i; + if (!PyList_Check(tmp)) { + PyErr_Format(PyExc_TypeError, "MatchMapping field \"patterns\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); + goto failed; + } + len = PyList_GET_SIZE(tmp); + patterns = _Py_asdl_pattern_seq_new(len, arena); + if (patterns == NULL) goto failed; + for (i = 0; i < len; i++) { + pattern_ty val; + PyObject *tmp2 = PyList_GET_ITEM(tmp, i); + Py_INCREF(tmp2); + res = obj2ast_pattern(state, tmp2, &val, arena); + Py_DECREF(tmp2); + if (res != 0) goto failed; + if (len != PyList_GET_SIZE(tmp)) { + PyErr_SetString(PyExc_RuntimeError, "MatchMapping field \"patterns\" changed size during iteration"); + goto failed; + } + asdl_seq_SET(patterns, i, val); + } + Py_CLEAR(tmp); + } + *out = _PyAST_MatchMapping(keys, patterns, lineno, col_offset, + end_lineno, end_col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } + tp = state->MatchClass_type; + isinstance = PyObject_IsInstance(obj, tp); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + expr_ty cls; + asdl_pattern_seq* patterns; + asdl_identifier_seq* extra_attrs; + asdl_pattern_seq* extra_patterns; + + if (_PyObject_LookupAttr(obj, state->cls, &tmp) < 0) { + return 1; + } + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"cls\" missing from MatchClass"); + return 1; + } + else { + int res; + res = obj2ast_expr(state, tmp, &cls, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + if (_PyObject_LookupAttr(obj, state->patterns, &tmp) < 0) { + return 1; + } + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"patterns\" missing from MatchClass"); + return 1; + } + else { + int res; + Py_ssize_t len; + Py_ssize_t i; + if (!PyList_Check(tmp)) { + PyErr_Format(PyExc_TypeError, "MatchClass field \"patterns\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); + goto failed; + } + len = PyList_GET_SIZE(tmp); + patterns = _Py_asdl_pattern_seq_new(len, arena); + if (patterns == NULL) goto failed; + for (i = 0; i < len; i++) { + pattern_ty val; + PyObject *tmp2 = PyList_GET_ITEM(tmp, i); + Py_INCREF(tmp2); + res = obj2ast_pattern(state, tmp2, &val, arena); + Py_DECREF(tmp2); + if (res != 0) goto failed; + if (len != PyList_GET_SIZE(tmp)) { + PyErr_SetString(PyExc_RuntimeError, "MatchClass field \"patterns\" changed size during iteration"); + goto failed; + } + asdl_seq_SET(patterns, i, val); + } + Py_CLEAR(tmp); + } + if (_PyObject_LookupAttr(obj, state->extra_attrs, &tmp) < 0) { + return 1; + } + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"extra_attrs\" missing from MatchClass"); + return 1; + } + else { + int res; + Py_ssize_t len; + Py_ssize_t i; + if (!PyList_Check(tmp)) { + PyErr_Format(PyExc_TypeError, "MatchClass field \"extra_attrs\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); + goto failed; + } + len = PyList_GET_SIZE(tmp); + extra_attrs = _Py_asdl_identifier_seq_new(len, arena); + if (extra_attrs == NULL) goto failed; + for (i = 0; i < len; i++) { + identifier val; + PyObject *tmp2 = PyList_GET_ITEM(tmp, i); + Py_INCREF(tmp2); + res = obj2ast_identifier(state, tmp2, &val, arena); + Py_DECREF(tmp2); + if (res != 0) goto failed; + if (len != PyList_GET_SIZE(tmp)) { + PyErr_SetString(PyExc_RuntimeError, "MatchClass field \"extra_attrs\" changed size during iteration"); + goto failed; + } + asdl_seq_SET(extra_attrs, i, val); + } + Py_CLEAR(tmp); + } + if (_PyObject_LookupAttr(obj, state->extra_patterns, &tmp) < 0) { + return 1; + } + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"extra_patterns\" missing from MatchClass"); + return 1; + } + else { + int res; + Py_ssize_t len; + Py_ssize_t i; + if (!PyList_Check(tmp)) { + PyErr_Format(PyExc_TypeError, "MatchClass field \"extra_patterns\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); + goto failed; + } + len = PyList_GET_SIZE(tmp); + extra_patterns = _Py_asdl_pattern_seq_new(len, arena); + if (extra_patterns == NULL) goto failed; + for (i = 0; i < len; i++) { + pattern_ty val; + PyObject *tmp2 = PyList_GET_ITEM(tmp, i); + Py_INCREF(tmp2); + res = obj2ast_pattern(state, tmp2, &val, arena); + Py_DECREF(tmp2); + if (res != 0) goto failed; + if (len != PyList_GET_SIZE(tmp)) { + PyErr_SetString(PyExc_RuntimeError, "MatchClass field \"extra_patterns\" changed size during iteration"); + goto failed; + } + asdl_seq_SET(extra_patterns, i, val); + } + Py_CLEAR(tmp); + } + *out = _PyAST_MatchClass(cls, patterns, extra_attrs, extra_patterns, + lineno, col_offset, end_lineno, + end_col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } + tp = state->MatchRestOfSequence_type; + isinstance = PyObject_IsInstance(obj, tp); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + identifier target; + + if (_PyObject_LookupAttr(obj, state->target, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + target = NULL; + } + else { + int res; + res = obj2ast_identifier(state, tmp, &target, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = _PyAST_MatchRestOfSequence(target, lineno, col_offset, + end_lineno, end_col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } + tp = state->MatchAs_type; + isinstance = PyObject_IsInstance(obj, tp); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + pattern_ty pattern; + identifier target; + + if (_PyObject_LookupAttr(obj, state->pattern, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + pattern = NULL; + } + else { + int res; + res = obj2ast_pattern(state, tmp, &pattern, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + if (_PyObject_LookupAttr(obj, state->target, &tmp) < 0) { + return 1; + } + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from MatchAs"); + return 1; + } + else { + int res; + res = obj2ast_identifier(state, tmp, &target, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = _PyAST_MatchAs(pattern, target, lineno, col_offset, end_lineno, + end_col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } + tp = state->MatchOr_type; + isinstance = PyObject_IsInstance(obj, tp); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + asdl_pattern_seq* patterns; + + if (_PyObject_LookupAttr(obj, state->patterns, &tmp) < 0) { + return 1; + } + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"patterns\" missing from MatchOr"); + return 1; + } + else { + int res; + Py_ssize_t len; + Py_ssize_t i; + if (!PyList_Check(tmp)) { + PyErr_Format(PyExc_TypeError, "MatchOr field \"patterns\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); + goto failed; + } + len = PyList_GET_SIZE(tmp); + patterns = _Py_asdl_pattern_seq_new(len, arena); + if (patterns == NULL) goto failed; + for (i = 0; i < len; i++) { + pattern_ty val; + PyObject *tmp2 = PyList_GET_ITEM(tmp, i); + Py_INCREF(tmp2); + res = obj2ast_pattern(state, tmp2, &val, arena); + Py_DECREF(tmp2); + if (res != 0) goto failed; + if (len != PyList_GET_SIZE(tmp)) { + PyErr_SetString(PyExc_RuntimeError, "MatchOr field \"patterns\" changed size during iteration"); + goto failed; + } + asdl_seq_SET(patterns, i, val); + } + Py_CLEAR(tmp); + } + *out = _PyAST_MatchOr(patterns, lineno, col_offset, end_lineno, + end_col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } + + PyErr_Format(PyExc_TypeError, "expected some sort of pattern, but got %R", obj); + failed: + Py_XDECREF(tmp); + return 1; +} + +int +obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* + out, PyArena* arena) +{ + int isinstance; + + PyObject *tmp = NULL; + PyObject *tp; + + if (obj == Py_None) { + *out = NULL; + return 0; + } + tp = state->TypeIgnore_type; + isinstance = PyObject_IsInstance(obj, tp); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + int lineno; + string tag; + + if (_PyObject_LookupAttr(obj, state->lineno, &tmp) < 0) { + return 1; + } + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from TypeIgnore"); + return 1; + } + else { + int res; + res = obj2ast_int(state, tmp, &lineno, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + if (_PyObject_LookupAttr(obj, state->tag, &tmp) < 0) { + return 1; + } + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"tag\" missing from TypeIgnore"); + return 1; + } + else { + int res; + res = obj2ast_string(state, tmp, &tag, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = _PyAST_TypeIgnore(lineno, tag, arena); + if (*out == NULL) goto failed; return 0; } @@ -10230,12 +11016,6 @@ astmodule_exec(PyObject *m) if (PyModule_AddObjectRef(m, "Slice", state->Slice_type) < 0) { return -1; } - if (PyModule_AddObjectRef(m, "MatchAs", state->MatchAs_type) < 0) { - return -1; - } - if (PyModule_AddObjectRef(m, "MatchOr", state->MatchOr_type) < 0) { - return -1; - } if (PyModule_AddObjectRef(m, "expr_context", state->expr_context_type) < 0) { return -1; @@ -10378,6 +11158,40 @@ astmodule_exec(PyObject *m) if (PyModule_AddObjectRef(m, "match_case", state->match_case_type) < 0) { return -1; } + if (PyModule_AddObjectRef(m, "pattern", state->pattern_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "MatchAlways", state->MatchAlways_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "MatchValue", state->MatchValue_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "MatchConstant", state->MatchConstant_type) < + 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "MatchSequence", state->MatchSequence_type) < + 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "MatchMapping", state->MatchMapping_type) < 0) + { + return -1; + } + if (PyModule_AddObjectRef(m, "MatchClass", state->MatchClass_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "MatchRestOfSequence", + state->MatchRestOfSequence_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "MatchAs", state->MatchAs_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "MatchOr", state->MatchOr_type) < 0) { + return -1; + } if (PyModule_AddObjectRef(m, "type_ignore", state->type_ignore_type) < 0) { return -1; } From 2d25ff78cded13238c6eb999d0f3be7ce283d188 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Thu, 22 Apr 2021 21:30:04 +1000 Subject: [PATCH 02/38] Everything except symtable.c now compiles --- Grammar/python.gram | 6 +- Include/internal/pycore_ast.h | 23 +- Include/internal/pycore_ast_state.h | 6 +- Parser/Python.asdl | 4 +- Parser/parser.c | 14 +- Parser/pegen.c | 2 +- Python/Python-ast.c | 107 +++++---- Python/ast.c | 70 +++++- Python/ast_opt.c | 118 ++-------- Python/compile.c | 338 ++++++++++++++++------------ 10 files changed, 355 insertions(+), 333 deletions(-) diff --git a/Grammar/python.gram b/Grammar/python.gram index 7422e0caace994..a194c367961e2f 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -318,9 +318,9 @@ maybe_star_pattern[pattern_ty]: | pattern star_pattern[pattern_ty]: | '*' target=pattern_capture_target { - _PyAST_MatchRestOfSequence(target->v.Name.id, EXTRA) } + _PyAST_MatchStar(target->v.Name.id, EXTRA) } | '*' wildcard_pattern { - _PyAST_MatchRestOfSequence(NULL, EXTRA) } + _PyAST_MatchStar(NULL, EXTRA) } mapping_pattern[pattern_ty]: | '{' items=items_pattern? '}' { @@ -362,7 +362,7 @@ positional_patterns[asdl_pattern_seq*]: | args[asdl_pattern_seq*]=','.pattern+ { args } keyword_patterns[asdl_seq*]: | keywords[asdl_seq*]=','.keyword_pattern+ { keywords } -keyword_pattern[pattern_ty]: +keyword_pattern[KeyPatternPair*]: | arg=NAME '=' value=pattern { _PyPegen_key_pattern_pair(p, arg, value) } return_stmt[stmt_ty]: diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index 9b22cd3bddd4f7..7726aa6ab7017c 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -561,9 +561,8 @@ struct _match_case { enum _pattern_kind {MatchAlways_kind=1, MatchValue_kind=2, MatchConstant_kind=3, MatchSequence_kind=4, - MatchMapping_kind=5, MatchClass_kind=6, - MatchRestOfSequence_kind=7, MatchAs_kind=8, - MatchOr_kind=9}; + MatchMapping_kind=5, MatchClass_kind=6, MatchStar_kind=7, + MatchAs_kind=8, MatchOr_kind=9}; struct _pattern { enum _pattern_kind kind; union { @@ -587,13 +586,13 @@ struct _pattern { struct { expr_ty cls; asdl_pattern_seq *patterns; - asdl_identifier_seq *extra_attrs; - asdl_pattern_seq *extra_patterns; + asdl_identifier_seq *kwd_attrs; + asdl_pattern_seq *kwd_patterns; } MatchClass; struct { identifier target; - } MatchRestOfSequence; + } MatchStar; struct { pattern_ty pattern; @@ -821,13 +820,11 @@ pattern_ty _PyAST_MatchMapping(asdl_expr_seq * keys, asdl_pattern_seq * patterns, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); pattern_ty _PyAST_MatchClass(expr_ty cls, asdl_pattern_seq * patterns, - asdl_identifier_seq * extra_attrs, - asdl_pattern_seq * extra_patterns, int lineno, int - col_offset, int end_lineno, int end_col_offset, - PyArena *arena); -pattern_ty _PyAST_MatchRestOfSequence(identifier target, int lineno, int - col_offset, int end_lineno, int - end_col_offset, PyArena *arena); + asdl_identifier_seq * kwd_attrs, asdl_pattern_seq + * kwd_patterns, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +pattern_ty _PyAST_MatchStar(identifier target, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); pattern_ty _PyAST_MatchAs(pattern_ty pattern, identifier target, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index 457e51c4b08c4e..4bdde03c029fd6 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -97,8 +97,8 @@ struct ast_state { PyObject *MatchConstant_type; PyObject *MatchMapping_type; PyObject *MatchOr_type; - PyObject *MatchRestOfSequence_type; PyObject *MatchSequence_type; + PyObject *MatchStar_type; PyObject *MatchValue_type; PyObject *Match_type; PyObject *Mod_singleton; @@ -184,8 +184,6 @@ struct ast_state { PyObject *excepthandler_type; PyObject *expr_context_type; PyObject *expr_type; - PyObject *extra_attrs; - PyObject *extra_patterns; PyObject *finalbody; PyObject *format_spec; PyObject *func; @@ -204,6 +202,8 @@ struct ast_state { PyObject *kind; PyObject *kw_defaults; PyObject *kwarg; + PyObject *kwd_attrs; + PyObject *kwd_patterns; PyObject *kwonlyargs; PyObject *left; PyObject *level; diff --git a/Parser/Python.asdl b/Parser/Python.asdl index 5b98cdf5b8751e..cac812dc3f6d90 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -131,9 +131,9 @@ module Python | MatchConstant(constant value) | MatchSequence(pattern* patterns) | MatchMapping(expr* keys, pattern* patterns) - | MatchClass(expr cls, pattern* patterns, identifier* extra_attrs, pattern* extra_patterns) + | MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns) - | MatchRestOfSequence(identifier? target) + | MatchStar(identifier? target) -- A NULL entry in the MatchMapping key list handles capturing extra mapping keys | MatchAs(pattern? pattern, identifier target) diff --git a/Parser/parser.c b/Parser/parser.c index f7cff9dcb2e367..3a10f82e8b7f2b 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -559,7 +559,7 @@ static KeyPatternPair* double_star_pattern_rule(Parser *p); static pattern_ty class_pattern_rule(Parser *p); static asdl_pattern_seq* positional_patterns_rule(Parser *p); static asdl_seq* keyword_patterns_rule(Parser *p); -static pattern_ty keyword_pattern_rule(Parser *p); +static KeyPatternPair* keyword_pattern_rule(Parser *p); static stmt_ty return_stmt_rule(Parser *p); static stmt_ty raise_stmt_rule(Parser *p); static stmt_ty function_def_rule(Parser *p); @@ -7218,7 +7218,7 @@ star_pattern_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_MatchRestOfSequence ( target -> v . Name . id , EXTRA ); + _res = _PyAST_MatchStar ( target -> v . Name . id , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -7254,7 +7254,7 @@ star_pattern_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_MatchRestOfSequence ( NULL , EXTRA ); + _res = _PyAST_MatchStar ( NULL , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -7806,7 +7806,7 @@ keyword_patterns_rule(Parser *p) } // keyword_pattern: NAME '=' pattern -static pattern_ty +static KeyPatternPair* keyword_pattern_rule(Parser *p) { D(p->level++); @@ -7814,7 +7814,7 @@ keyword_pattern_rule(Parser *p) D(p->level--); return NULL; } - pattern_ty _res = NULL; + KeyPatternPair* _res = NULL; int _mark = p->mark; { // NAME '=' pattern if (p->error_indicator) { @@ -24474,7 +24474,7 @@ _loop0_65_rule(Parser *p) } D(fprintf(stderr, "%*c> _loop0_65[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' keyword_pattern")); Token * _literal; - pattern_ty elem; + KeyPatternPair* elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -24538,7 +24538,7 @@ _gather_64_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _gather_64[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "keyword_pattern _loop0_65")); - pattern_ty elem; + KeyPatternPair* elem; asdl_seq * seq; if ( (elem = keyword_pattern_rule(p)) // keyword_pattern diff --git a/Parser/pegen.c b/Parser/pegen.c index e147cedba62577..aa948eed85ea69 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -1794,7 +1794,7 @@ _PyPegen_get_values(Parser *p, asdl_seq *seq) return new_seq; } -/* Constructs a KeyPatternPair that is used when parsing a mapping pattern */ +/* Constructs a KeyPatternPair that is used when parsing mapping & class patterns */ KeyPatternPair * _PyPegen_key_pattern_pair(Parser *p, expr_ty key, pattern_ty pattern) { diff --git a/Python/Python-ast.c b/Python/Python-ast.c index bf435432900f6d..ea63245cf3bdb2 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -111,8 +111,8 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(state->MatchConstant_type); Py_CLEAR(state->MatchMapping_type); Py_CLEAR(state->MatchOr_type); - Py_CLEAR(state->MatchRestOfSequence_type); Py_CLEAR(state->MatchSequence_type); + Py_CLEAR(state->MatchStar_type); Py_CLEAR(state->MatchValue_type); Py_CLEAR(state->Match_type); Py_CLEAR(state->Mod_singleton); @@ -198,8 +198,6 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(state->excepthandler_type); Py_CLEAR(state->expr_context_type); Py_CLEAR(state->expr_type); - Py_CLEAR(state->extra_attrs); - Py_CLEAR(state->extra_patterns); Py_CLEAR(state->finalbody); Py_CLEAR(state->format_spec); Py_CLEAR(state->func); @@ -218,6 +216,8 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(state->kind); Py_CLEAR(state->kw_defaults); Py_CLEAR(state->kwarg); + Py_CLEAR(state->kwd_attrs); + Py_CLEAR(state->kwd_patterns); Py_CLEAR(state->kwonlyargs); Py_CLEAR(state->left); Py_CLEAR(state->level); @@ -300,8 +300,6 @@ static int init_identifiers(struct ast_state *state) if ((state->end_col_offset = PyUnicode_InternFromString("end_col_offset")) == NULL) return 0; if ((state->end_lineno = PyUnicode_InternFromString("end_lineno")) == NULL) return 0; if ((state->exc = PyUnicode_InternFromString("exc")) == NULL) return 0; - if ((state->extra_attrs = PyUnicode_InternFromString("extra_attrs")) == NULL) return 0; - if ((state->extra_patterns = PyUnicode_InternFromString("extra_patterns")) == NULL) return 0; if ((state->finalbody = PyUnicode_InternFromString("finalbody")) == NULL) return 0; if ((state->format_spec = PyUnicode_InternFromString("format_spec")) == NULL) return 0; if ((state->func = PyUnicode_InternFromString("func")) == NULL) return 0; @@ -319,6 +317,8 @@ static int init_identifiers(struct ast_state *state) if ((state->kind = PyUnicode_InternFromString("kind")) == NULL) return 0; if ((state->kw_defaults = PyUnicode_InternFromString("kw_defaults")) == NULL) return 0; if ((state->kwarg = PyUnicode_InternFromString("kwarg")) == NULL) return 0; + if ((state->kwd_attrs = PyUnicode_InternFromString("kwd_attrs")) == NULL) return 0; + if ((state->kwd_patterns = PyUnicode_InternFromString("kwd_patterns")) == NULL) return 0; if ((state->kwonlyargs = PyUnicode_InternFromString("kwonlyargs")) == NULL) return 0; if ((state->left = PyUnicode_InternFromString("left")) == NULL) return 0; if ((state->level = PyUnicode_InternFromString("level")) == NULL) return 0; @@ -727,10 +727,10 @@ static const char * const MatchMapping_fields[]={ static const char * const MatchClass_fields[]={ "cls", "patterns", - "extra_attrs", - "extra_patterns", + "kwd_attrs", + "kwd_patterns", }; -static const char * const MatchRestOfSequence_fields[]={ +static const char * const MatchStar_fields[]={ "target", }; static const char * const MatchAs_fields[]={ @@ -1777,8 +1777,8 @@ init_types(struct ast_state *state) " | MatchConstant(constant value)\n" " | MatchSequence(pattern* patterns)\n" " | MatchMapping(expr* keys, pattern* patterns)\n" - " | MatchClass(expr cls, pattern* patterns, identifier* extra_attrs, pattern* extra_patterns)\n" - " | MatchRestOfSequence(identifier? target)\n" + " | MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns)\n" + " | MatchStar(identifier? target)\n" " | MatchAs(pattern? pattern, identifier target)\n" " | MatchOr(pattern* patterns)"); if (!state->pattern_type) return 0; @@ -1816,15 +1816,13 @@ init_types(struct ast_state *state) state->MatchClass_type = make_type(state, "MatchClass", state->pattern_type, MatchClass_fields, 4, - "MatchClass(expr cls, pattern* patterns, identifier* extra_attrs, pattern* extra_patterns)"); + "MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns)"); if (!state->MatchClass_type) return 0; - state->MatchRestOfSequence_type = make_type(state, "MatchRestOfSequence", - state->pattern_type, - MatchRestOfSequence_fields, 1, - "MatchRestOfSequence(identifier? target)"); - if (!state->MatchRestOfSequence_type) return 0; - if (PyObject_SetAttr(state->MatchRestOfSequence_type, state->target, - Py_None) == -1) + state->MatchStar_type = make_type(state, "MatchStar", state->pattern_type, + MatchStar_fields, 1, + "MatchStar(identifier? target)"); + if (!state->MatchStar_type) return 0; + if (PyObject_SetAttr(state->MatchStar_type, state->target, Py_None) == -1) return 0; state->MatchAs_type = make_type(state, "MatchAs", state->pattern_type, MatchAs_fields, 2, @@ -3494,8 +3492,8 @@ _PyAST_MatchMapping(asdl_expr_seq * keys, asdl_pattern_seq * patterns, int pattern_ty _PyAST_MatchClass(expr_ty cls, asdl_pattern_seq * patterns, asdl_identifier_seq - * extra_attrs, asdl_pattern_seq * extra_patterns, int lineno, - int col_offset, int end_lineno, int end_col_offset, PyArena + * kwd_attrs, asdl_pattern_seq * kwd_patterns, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena *arena) { pattern_ty p; @@ -3510,8 +3508,8 @@ _PyAST_MatchClass(expr_ty cls, asdl_pattern_seq * patterns, asdl_identifier_seq p->kind = MatchClass_kind; p->v.MatchClass.cls = cls; p->v.MatchClass.patterns = patterns; - p->v.MatchClass.extra_attrs = extra_attrs; - p->v.MatchClass.extra_patterns = extra_patterns; + p->v.MatchClass.kwd_attrs = kwd_attrs; + p->v.MatchClass.kwd_patterns = kwd_patterns; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -3520,15 +3518,15 @@ _PyAST_MatchClass(expr_ty cls, asdl_pattern_seq * patterns, asdl_identifier_seq } pattern_ty -_PyAST_MatchRestOfSequence(identifier target, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena) +_PyAST_MatchStar(identifier target, int lineno, int col_offset, int end_lineno, + int end_col_offset, PyArena *arena) { pattern_ty p; p = (pattern_ty)_PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; - p->kind = MatchRestOfSequence_kind; - p->v.MatchRestOfSequence.target = target; + p->kind = MatchStar_kind; + p->v.MatchStar.target = target; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -5245,24 +5243,24 @@ ast2obj_pattern(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->patterns, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.MatchClass.extra_attrs, + value = ast2obj_list(state, (asdl_seq*)o->v.MatchClass.kwd_attrs, ast2obj_identifier); if (!value) goto failed; - if (PyObject_SetAttr(result, state->extra_attrs, value) == -1) + if (PyObject_SetAttr(result, state->kwd_attrs, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(state, (asdl_seq*)o->v.MatchClass.extra_patterns, + value = ast2obj_list(state, (asdl_seq*)o->v.MatchClass.kwd_patterns, ast2obj_pattern); if (!value) goto failed; - if (PyObject_SetAttr(result, state->extra_patterns, value) == -1) + if (PyObject_SetAttr(result, state->kwd_patterns, value) == -1) goto failed; Py_DECREF(value); break; - case MatchRestOfSequence_kind: - tp = (PyTypeObject *)state->MatchRestOfSequence_type; + case MatchStar_kind: + tp = (PyTypeObject *)state->MatchStar_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.MatchRestOfSequence.target); + value = ast2obj_identifier(state, o->v.MatchStar.target); if (!value) goto failed; if (PyObject_SetAttr(result, state->target, value) == -1) goto failed; @@ -10513,8 +10511,8 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (isinstance) { expr_ty cls; asdl_pattern_seq* patterns; - asdl_identifier_seq* extra_attrs; - asdl_pattern_seq* extra_patterns; + asdl_identifier_seq* kwd_attrs; + asdl_pattern_seq* kwd_patterns; if (_PyObject_LookupAttr(obj, state->cls, &tmp) < 0) { return 1; @@ -10562,11 +10560,11 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, } Py_CLEAR(tmp); } - if (_PyObject_LookupAttr(obj, state->extra_attrs, &tmp) < 0) { + if (_PyObject_LookupAttr(obj, state->kwd_attrs, &tmp) < 0) { return 1; } if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"extra_attrs\" missing from MatchClass"); + PyErr_SetString(PyExc_TypeError, "required field \"kwd_attrs\" missing from MatchClass"); return 1; } else { @@ -10574,12 +10572,12 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, Py_ssize_t len; Py_ssize_t i; if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "MatchClass field \"extra_attrs\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); + PyErr_Format(PyExc_TypeError, "MatchClass field \"kwd_attrs\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); goto failed; } len = PyList_GET_SIZE(tmp); - extra_attrs = _Py_asdl_identifier_seq_new(len, arena); - if (extra_attrs == NULL) goto failed; + kwd_attrs = _Py_asdl_identifier_seq_new(len, arena); + if (kwd_attrs == NULL) goto failed; for (i = 0; i < len; i++) { identifier val; PyObject *tmp2 = PyList_GET_ITEM(tmp, i); @@ -10588,18 +10586,18 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, Py_DECREF(tmp2); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { - PyErr_SetString(PyExc_RuntimeError, "MatchClass field \"extra_attrs\" changed size during iteration"); + PyErr_SetString(PyExc_RuntimeError, "MatchClass field \"kwd_attrs\" changed size during iteration"); goto failed; } - asdl_seq_SET(extra_attrs, i, val); + asdl_seq_SET(kwd_attrs, i, val); } Py_CLEAR(tmp); } - if (_PyObject_LookupAttr(obj, state->extra_patterns, &tmp) < 0) { + if (_PyObject_LookupAttr(obj, state->kwd_patterns, &tmp) < 0) { return 1; } if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"extra_patterns\" missing from MatchClass"); + PyErr_SetString(PyExc_TypeError, "required field \"kwd_patterns\" missing from MatchClass"); return 1; } else { @@ -10607,12 +10605,12 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, Py_ssize_t len; Py_ssize_t i; if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "MatchClass field \"extra_patterns\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); + PyErr_Format(PyExc_TypeError, "MatchClass field \"kwd_patterns\" must be a list, not a %.200s", _PyType_Name(Py_TYPE(tmp))); goto failed; } len = PyList_GET_SIZE(tmp); - extra_patterns = _Py_asdl_pattern_seq_new(len, arena); - if (extra_patterns == NULL) goto failed; + kwd_patterns = _Py_asdl_pattern_seq_new(len, arena); + if (kwd_patterns == NULL) goto failed; for (i = 0; i < len; i++) { pattern_ty val; PyObject *tmp2 = PyList_GET_ITEM(tmp, i); @@ -10621,20 +10619,20 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, Py_DECREF(tmp2); if (res != 0) goto failed; if (len != PyList_GET_SIZE(tmp)) { - PyErr_SetString(PyExc_RuntimeError, "MatchClass field \"extra_patterns\" changed size during iteration"); + PyErr_SetString(PyExc_RuntimeError, "MatchClass field \"kwd_patterns\" changed size during iteration"); goto failed; } - asdl_seq_SET(extra_patterns, i, val); + asdl_seq_SET(kwd_patterns, i, val); } Py_CLEAR(tmp); } - *out = _PyAST_MatchClass(cls, patterns, extra_attrs, extra_patterns, + *out = _PyAST_MatchClass(cls, patterns, kwd_attrs, kwd_patterns, lineno, col_offset, end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } - tp = state->MatchRestOfSequence_type; + tp = state->MatchStar_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { return 1; @@ -10655,8 +10653,8 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = _PyAST_MatchRestOfSequence(target, lineno, col_offset, - end_lineno, end_col_offset, arena); + *out = _PyAST_MatchStar(target, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -11182,8 +11180,7 @@ astmodule_exec(PyObject *m) if (PyModule_AddObjectRef(m, "MatchClass", state->MatchClass_type) < 0) { return -1; } - if (PyModule_AddObjectRef(m, "MatchRestOfSequence", - state->MatchRestOfSequence_type) < 0) { + if (PyModule_AddObjectRef(m, "MatchStar", state->MatchStar_type) < 0) { return -1; } if (PyModule_AddObjectRef(m, "MatchAs", state->MatchAs_type) < 0) { diff --git a/Python/ast.c b/Python/ast.c index c87795305e507f..f07431660b0c19 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -308,14 +308,6 @@ validate_expr(expr_ty exp, expr_context_ty ctx) return validate_exprs(exp->v.Tuple.elts, ctx, 0); case NamedExpr_kind: return validate_expr(exp->v.NamedExpr.value, Load); - case MatchAs_kind: - PyErr_SetString(PyExc_ValueError, - "MatchAs is only valid in match_case patterns"); - return 0; - case MatchOr_kind: - PyErr_SetString(PyExc_ValueError, - "MatchOr is only valid in match_case patterns"); - return 0; /* This last case doesn't have any checking. */ case Name_kind: return 1; @@ -325,10 +317,68 @@ validate_expr(expr_ty exp, expr_context_ty ctx) } static int -validate_pattern(expr_ty p) +validate_pattern(pattern_ty p) { // Coming soon (thanks Batuhan)! - return 1; + // TODO: Potentially ensure no subnodes use "_" as an ordinary identifier + switch (p->kind) { + case MatchAlways_kind: + // Nothing to check + return 1; + case MatchValue_kind: + // TODO: Check value is a constant or an attribute lookup + // (will need to allow selected unary ops & binops if + // validation is performed before constant folding...) + return validate_expr(p->v.MatchValue.value, Load); + case MatchConstant_kind: + // TODO: Check constant is specifically None, True, or False + return validate_constant(p->v.MatchConstant.value); + case MatchSequence_kind: + // TODO: Validate all subpatterns + // return validate_patterns(p->v.MatchSequence.patterns); + return 1; + case MatchMapping_kind: + if (asdl_seq_LEN(p->v.MatchMapping.keys) != asdl_seq_LEN(p->v.MatchMapping.patterns)) { + PyErr_SetString(PyExc_ValueError, + "MatchMapping doesn't have the same number of keys as patterns"); + return 0; + } + // null_ok=1 for key expressions to allow rest-of-mapping capture in patterns + // TODO: replace with more restrictive expression validator, as per MatchValue above + if (!validate_exprs(p->v.MatchMapping.keys, Load, /*null_ok=*/ 1)) { + return 0; + } + // TODO: Validate all subpatterns + // return validate_patterns(p->v.MatchMapping.patterns); + return 1; + case MatchClass_kind: + if (asdl_seq_LEN(p->v.MatchMapping.keys) != asdl_seq_LEN(p->v.MatchMapping.patterns)) { + PyErr_SetString(PyExc_ValueError, + "MatchMapping doesn't have the same number of keys as patterns"); + return 0; + } + // TODO: Restrict cls lookup to being a name or attribute + if (!validate_expr(p->v.MatchClass.cls, Load)) { + return 0; + } + // TODO: Validate all subpatterns + // return validate_patterns(p->v.MatchClass.patterns) && + // validate_patterns(p->v.MatchClass.kwd_patterns); + return 1; + case MatchStar_kind: + // Nothing to check (except to potentially block "_" as an identifer) + break; + case MatchAs_kind: + return validate_pattern(p->v.MatchAs.pattern); + case MatchOr_kind: + // TODO: Validate all subpatterns + // return validate_patterns(p->v.MatchOr.patterns); + return 1; + // No default case, so the compiler will emit a warning if new pattern + // kinds are added without being handled here + } + PyErr_SetString(PyExc_SystemError, "unexpected pattern"); + return 0; } static int diff --git a/Python/ast_opt.c b/Python/ast_opt.c index dea20da07e69d6..085ebd6fcc2464 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -410,7 +410,7 @@ static int astfold_arg(arg_ty node_, PyArena *ctx_, _PyASTOptimizeState *state); static int astfold_withitem(withitem_ty node_, PyArena *ctx_, _PyASTOptimizeState *state); static int astfold_excepthandler(excepthandler_ty node_, PyArena *ctx_, _PyASTOptimizeState *state); static int astfold_match_case(match_case_ty node_, PyArena *ctx_, _PyASTOptimizeState *state); -static int astfold_pattern(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state); +static int astfold_pattern(pattern_ty node_, PyArena *ctx_, _PyASTOptimizeState *state); #define CALL(FUNC, TYPE, ARG) \ if (!FUNC((ARG), ctx_, state)) \ @@ -595,10 +595,6 @@ astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) case Constant_kind: // Already a constant, nothing further to do break; - case MatchAs_kind: - case MatchOr_kind: - // These can't occur outside of patterns. - Py_UNREACHABLE(); // No default case, so the compiler will emit a warning if new expression // kinds are added without being handled here } @@ -783,111 +779,41 @@ astfold_withitem(withitem_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) } static int -astfold_pattern_negative(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) +astfold_pattern(pattern_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) { - assert(node_->kind == UnaryOp_kind); - assert(node_->v.UnaryOp.op == USub); - assert(node_->v.UnaryOp.operand->kind == Constant_kind); - PyObject *value = node_->v.UnaryOp.operand->v.Constant.value; - assert(PyComplex_CheckExact(value) || - PyFloat_CheckExact(value) || - PyLong_CheckExact(value)); - PyObject *negated = PyNumber_Negative(value); - if (negated == NULL) { - return 0; - } - assert(PyComplex_CheckExact(negated) || - PyFloat_CheckExact(negated) || - PyLong_CheckExact(negated)); - return make_const(node_, negated, ctx_); -} - -static int -astfold_pattern_complex(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) -{ - expr_ty left = node_->v.BinOp.left; - expr_ty right = node_->v.BinOp.right; - if (left->kind == UnaryOp_kind) { - CALL(astfold_pattern_negative, expr_ty, left); - } - assert(left->kind = Constant_kind); - assert(right->kind = Constant_kind); - // LHS must be real, RHS must be imaginary: - if (!(PyFloat_CheckExact(left->v.Constant.value) || - PyLong_CheckExact(left->v.Constant.value)) || - !PyComplex_CheckExact(right->v.Constant.value)) - { - // Not actually valid, but it's the compiler's job to complain: - return 1; - } - PyObject *new; - if (node_->v.BinOp.op == Add) { - new = PyNumber_Add(left->v.Constant.value, right->v.Constant.value); - } - else { - assert(node_->v.BinOp.op == Sub); - new = PyNumber_Subtract(left->v.Constant.value, right->v.Constant.value); - } - if (new == NULL) { - return 0; - } - assert(PyComplex_CheckExact(new)); - return make_const(node_, new, ctx_); -} - -static int -astfold_pattern_keyword(keyword_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) -{ - CALL(astfold_pattern, expr_ty, node_->value); - return 1; -} - -static int -astfold_pattern(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) -{ - // Don't blindly optimize the pattern as an expr; it plays by its own rules! - // Currently, this is only used to form complex/negative numeric constants. + // Currently, this is really only used to form complex/negative numeric + // constants in MatchValue and MatchMapping nodes + // We still recurse into all subexpressions and subpatterns anyway switch (node_->kind) { - case Attribute_kind: + case MatchAlways_kind: break; - case BinOp_kind: - CALL(astfold_pattern_complex, expr_ty, node_); + case MatchValue_kind: + CALL(astfold_expr, expr_ty, node_->v.MatchValue.value); break; - case Call_kind: - CALL_SEQ(astfold_pattern, expr, node_->v.Call.args); - CALL_SEQ(astfold_pattern_keyword, keyword, node_->v.Call.keywords); + case MatchConstant_kind: break; - case Constant_kind: + case MatchSequence_kind: + CALL_SEQ(astfold_pattern, pattern, node_->v.MatchSequence.patterns); break; - case Dict_kind: - CALL_SEQ(astfold_pattern, expr, node_->v.Dict.keys); - CALL_SEQ(astfold_pattern, expr, node_->v.Dict.values); + case MatchMapping_kind: + CALL_SEQ(astfold_expr, expr, node_->v.MatchMapping.keys); + CALL_SEQ(astfold_pattern, pattern, node_->v.MatchMapping.patterns); break; - // Not actually valid, but it's the compiler's job to complain: - case JoinedStr_kind: + case MatchClass_kind: + CALL(astfold_expr, expr_ty, node_->v.MatchClass.cls); + CALL_SEQ(astfold_pattern, pattern, node_->v.MatchClass.patterns); + CALL_SEQ(astfold_pattern, pattern, node_->v.MatchClass.kwd_patterns); break; - case List_kind: - CALL_SEQ(astfold_pattern, expr, node_->v.List.elts); + case MatchStar_kind: break; case MatchAs_kind: CALL(astfold_pattern, expr_ty, node_->v.MatchAs.pattern); break; case MatchOr_kind: - CALL_SEQ(astfold_pattern, expr, node_->v.MatchOr.patterns); - break; - case Name_kind: - break; - case Starred_kind: - CALL(astfold_pattern, expr_ty, node_->v.Starred.value); + CALL_SEQ(astfold_pattern, pattern, node_->v.MatchOr.patterns); break; - case Tuple_kind: - CALL_SEQ(astfold_pattern, expr, node_->v.Tuple.elts); - break; - case UnaryOp_kind: - CALL(astfold_pattern_negative, expr_ty, node_); - break; - default: - Py_UNREACHABLE(); + // No default case, so the compiler will emit a warning if new pattern + // kinds are added without being handled here } return 1; } diff --git a/Python/compile.c b/Python/compile.c index 49a713b2b0c36d..9eb4e901b9c0cc 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -278,9 +278,9 @@ static int compiler_async_comprehension_generator( int depth, expr_ty elt, expr_ty val, int type); -static int compiler_pattern(struct compiler *, expr_ty, pattern_context *); +static int compiler_pattern(struct compiler *, pattern_ty, pattern_context *); static int compiler_match(struct compiler *, stmt_ty); -static int compiler_pattern_subpattern(struct compiler *, expr_ty, +static int compiler_pattern_subpattern(struct compiler *, pattern_ty, pattern_context *); static PyCodeObject *assemble(struct compiler *, int addNone); @@ -5257,10 +5257,6 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) return compiler_list(c, e); case Tuple_kind: return compiler_tuple(c, e); - case MatchAs_kind: - case MatchOr_kind: - // Can only occur in patterns, which are handled elsewhere. - Py_UNREACHABLE(); } return 1; } @@ -5587,16 +5583,18 @@ compiler_slice(struct compiler *c, expr_ty s) // that it's much easier to smooth out any redundant pushing, popping, and // jumping in the peephole optimizer than to detect or predict it here. - #define WILDCARD_CHECK(N) \ - ((N)->kind == Name_kind && \ - _PyUnicode_EqualToASCIIString((N)->v.Name.id, "_")) + ((N)->kind == MatchAlways_kind) +#define WILDCARD_STAR_CHECK(N) \ + ((N)->kind == MatchStar_kind && !(N)->v.MatchStar.target) static int pattern_helper_store_name(struct compiler *c, identifier n, pattern_context *pc) { - assert(!_PyUnicode_EqualToASCIIString(n, "_")); + if (forbidden_name(c, n, Store)) { + return 0; + } // Can't assign to the same name twice: if (pc->stores == NULL) { RETURN_IF_FALSE(pc->stores = PySet_New(NULL)); @@ -5618,16 +5616,43 @@ pattern_helper_store_name(struct compiler *c, identifier n, pattern_context *pc) static int -pattern_helper_sequence_unpack(struct compiler *c, asdl_expr_seq *values, +pattern_unpack_helper(struct compiler *c, asdl_pattern_seq *elts) +{ + Py_ssize_t n = asdl_seq_LEN(elts); + int seen_star = 0; + for (Py_ssize_t i = 0; i < n; i++) { + pattern_ty elt = asdl_seq_GET(elts, i); + if (elt->kind == MatchStar_kind && !seen_star) { + if ((i >= (1 << 8)) || + (n-i-1 >= (INT_MAX >> 8))) + return compiler_error(c, + "too many expressions in " + "star-unpacking sequence pattern"); + ADDOP_I(c, UNPACK_EX, (i + ((n-i-1) << 8))); + seen_star = 1; + } + else if (elt->kind == MatchStar_kind) { + return compiler_error(c, + "multiple starred expressions in sequence pattern"); + } + } + if (!seen_star) { + ADDOP_I(c, UNPACK_SEQUENCE, n); + } + return 1; +} + +static int +pattern_helper_sequence_unpack(struct compiler *c, asdl_pattern_seq *patterns, Py_ssize_t star, pattern_context *pc) { - RETURN_IF_FALSE(unpack_helper(c, values)); + RETURN_IF_FALSE(pattern_unpack_helper(c, patterns)); // We've now got a bunch of new subjects on the stack. If any of them fail // to match, we need to pop everything else off, then finally push False. // fails is an array of blocks that correspond to the necessary amount of // popping for each element: basicblock **fails; - Py_ssize_t size = asdl_seq_LEN(values); + Py_ssize_t size = asdl_seq_LEN(patterns); fails = (basicblock **)PyObject_Malloc(sizeof(basicblock*) * size); if (fails == NULL) { PyErr_NoMemory(); @@ -5642,12 +5667,9 @@ pattern_helper_sequence_unpack(struct compiler *c, asdl_expr_seq *values, } } for (Py_ssize_t i = 0; i < size; i++) { - expr_ty value = asdl_seq_GET(values, i); - if (i == star) { - assert(value->kind == Starred_kind); - value = value->v.Starred.value; - } - if (!compiler_pattern_subpattern(c, value, pc) || + pattern_ty pattern = asdl_seq_GET(patterns, i); + assert(i != star || pattern->kind == MatchStar_kind); + if (!compiler_pattern_subpattern(c, pattern, pc) || !compiler_addop_j(c, POP_JUMP_IF_FALSE, fails[i]) || compiler_next_block(c) == NULL) { @@ -5690,21 +5712,20 @@ pattern_helper_sequence_unpack(struct compiler *c, asdl_expr_seq *values, // UNPACK_SEQUENCE / UNPACK_EX. This is more efficient for patterns with a // starred wildcard like [first, *_] / [first, *_, last] / [*_, last] / etc. static int -pattern_helper_sequence_subscr(struct compiler *c, asdl_expr_seq *values, +pattern_helper_sequence_subscr(struct compiler *c, asdl_pattern_seq *patterns, Py_ssize_t star, pattern_context *pc) { basicblock *end, *fail_pop_1; RETURN_IF_FALSE(end = compiler_new_block(c)); RETURN_IF_FALSE(fail_pop_1 = compiler_new_block(c)); - Py_ssize_t size = asdl_seq_LEN(values); + Py_ssize_t size = asdl_seq_LEN(patterns); for (Py_ssize_t i = 0; i < size; i++) { - expr_ty value = asdl_seq_GET(values, i); - if (WILDCARD_CHECK(value)) { + pattern_ty pattern = asdl_seq_GET(patterns, i); + if (WILDCARD_CHECK(pattern)) { continue; } if (i == star) { - assert(value->kind == Starred_kind); - assert(WILDCARD_CHECK(value->v.Starred.value)); + assert(WILDCARD_STAR_CHECK(pattern)); continue; } ADDOP(c, DUP_TOP); @@ -5719,7 +5740,7 @@ pattern_helper_sequence_subscr(struct compiler *c, asdl_expr_seq *values, ADDOP(c, BINARY_SUBTRACT); } ADDOP(c, BINARY_SUBSCR); - RETURN_IF_FALSE(compiler_pattern_subpattern(c, value, pc)); + RETURN_IF_FALSE(compiler_pattern_subpattern(c, pattern, pc)); ADDOP_JUMP(c, POP_JUMP_IF_FALSE, fail_pop_1); NEXT_BLOCK(c); } @@ -5733,10 +5754,9 @@ pattern_helper_sequence_subscr(struct compiler *c, asdl_expr_seq *values, return 1; } - // Like compiler_pattern, but turn off checks for irrefutability. static int -compiler_pattern_subpattern(struct compiler *c, expr_ty p, pattern_context *pc) +compiler_pattern_subpattern(struct compiler *c, pattern_ty p, pattern_context *pc) { int allow_irrefutable = pc->allow_irrefutable; pc->allow_irrefutable = 1; @@ -5745,11 +5765,26 @@ compiler_pattern_subpattern(struct compiler *c, expr_ty p, pattern_context *pc) return 1; } +static int +compiler_pattern_capture(struct compiler *c, identifier n, pattern_context *pc) +{ + RETURN_IF_FALSE(pattern_helper_store_name(c, n, pc)); + ADDOP_LOAD_CONST(c, Py_True); + return 1; +} static int -compiler_pattern_as(struct compiler *c, expr_ty p, pattern_context *pc) +compiler_pattern_as(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchAs_kind); + if (p->v.MatchAs.pattern == NULL) { + if (!pc->allow_irrefutable) { + // Whoops, can't have a name capture here! + const char *e = "name capture %R makes remaining patterns unreachable"; + return compiler_error(c, e, p->v.MatchAs.target); + } + return compiler_pattern_capture(c, p->v.MatchAs.target, pc); + } basicblock *end, *fail_pop_1; RETURN_IF_FALSE(end = compiler_new_block(c)); RETURN_IF_FALSE(fail_pop_1 = compiler_new_block(c)); @@ -5758,7 +5793,7 @@ compiler_pattern_as(struct compiler *c, expr_ty p, pattern_context *pc) RETURN_IF_FALSE(compiler_pattern(c, p->v.MatchAs.pattern, pc)); ADDOP_JUMP(c, POP_JUMP_IF_FALSE, fail_pop_1); NEXT_BLOCK(c); - RETURN_IF_FALSE(pattern_helper_store_name(c, p->v.MatchAs.name, pc)); + RETURN_IF_FALSE(pattern_helper_store_name(c, p->v.MatchAs.target, pc)); ADDOP_LOAD_CONST(c, Py_True); ADDOP_JUMP(c, JUMP_FORWARD, end); compiler_use_next_block(c, fail_pop_1); @@ -5769,71 +5804,101 @@ compiler_pattern_as(struct compiler *c, expr_ty p, pattern_context *pc) return 1; } - static int -compiler_pattern_capture(struct compiler *c, expr_ty p, pattern_context *pc) +compiler_pattern_star(struct compiler *c, pattern_ty p, pattern_context *pc) { - assert(p->kind == Name_kind); - assert(p->v.Name.ctx == Store); - assert(!WILDCARD_CHECK(p)); + assert(p->kind == MatchStar_kind); if (!pc->allow_irrefutable) { - // Whoops, can't have a name capture here! - const char *e = "name capture %R makes remaining patterns unreachable"; - return compiler_error(c, e, p->v.Name.id); + // Whoops, can't have a star capture here! + const char *e = "star captures are only allowed as part of sequence patterns"; + return compiler_error(c, e); } - RETURN_IF_FALSE(pattern_helper_store_name(c, p->v.Name.id, pc)); - ADDOP_LOAD_CONST(c, Py_True); - return 1; + return compiler_pattern_capture(c, p->v.MatchStar.target, pc); } +static int +validate_kwd_attrs(struct compiler *c, asdl_identifier_seq *attrs, asdl_pattern_seq* patterns) +{ + // Any errors will point to the pattern rather than the arg name as the + // parser is only supplying identifiers rather than Name or keyword nodes + Py_ssize_t nattrs = asdl_seq_LEN(attrs); + for (Py_ssize_t i = 0; i < nattrs; i++) { + identifier attr = ((identifier)asdl_seq_GET(attrs, i)); + c->u->u_col_offset = ((pattern_ty) asdl_seq_GET(patterns, i))->col_offset; + if (forbidden_name(c, attr, Store)) { + return -1; + } + for (Py_ssize_t j = i + 1; j < nattrs; j++) { + identifier other = ((identifier)asdl_seq_GET(attrs, j)); + if (!PyUnicode_Compare(attr, other)) { + c->u->u_col_offset = ((pattern_ty) asdl_seq_GET(patterns, j))->col_offset; + compiler_error(c, "attribute name repeated in class pattern: %U", attr); + return -1; + } + } + } + return 0; +} static int -compiler_pattern_class(struct compiler *c, expr_ty p, pattern_context *pc) +compiler_pattern_class(struct compiler *c, pattern_ty p, pattern_context *pc) { - asdl_expr_seq *args = p->v.Call.args; - asdl_keyword_seq *kwargs = p->v.Call.keywords; - Py_ssize_t nargs = asdl_seq_LEN(args); - Py_ssize_t nkwargs = asdl_seq_LEN(kwargs); - if (INT_MAX < nargs || INT_MAX < nargs + nkwargs - 1) { + assert(p->kind == MatchClass_kind); + asdl_pattern_seq *patterns = p->v.MatchClass.patterns; + asdl_identifier_seq *kwd_attrs = p->v.MatchClass.kwd_attrs; + asdl_pattern_seq *kwd_patterns = p->v.MatchClass.kwd_patterns; + Py_ssize_t nargs = patterns ? asdl_seq_LEN(patterns) : 0; + Py_ssize_t nattrs = kwd_attrs ? asdl_seq_LEN(kwd_attrs) : 0; + Py_ssize_t nkwd_patterns = kwd_patterns ? asdl_seq_LEN(kwd_patterns) : 0; + if (nattrs != nkwd_patterns) { + // AST validator shouldn't let this happen, but if it does, + // just fail, don't crash out of the interpreter + const char * e = "kwd_attrs (%d) / kwd_patterns (%d) length mismatch in class pattern"; + return compiler_error(c, e, nattrs, nkwd_patterns); + } + if (INT_MAX < nargs || INT_MAX < nargs + nattrs - 1) { const char *e = "too many sub-patterns in class pattern %R"; - return compiler_error(c, e, p->v.Call.func); + return compiler_error(c, e, p->v.MatchClass.cls); + } + if (nattrs) { + RETURN_IF_FALSE(!validate_kwd_attrs(c, kwd_attrs, kwd_patterns)); + c->u->u_col_offset = p->col_offset; // validate_kwd_attrs moves this } - RETURN_IF_FALSE(!validate_keywords(c, kwargs)); basicblock *end, *fail_pop_1; RETURN_IF_FALSE(end = compiler_new_block(c)); RETURN_IF_FALSE(fail_pop_1 = compiler_new_block(c)); - VISIT(c, expr, p->v.Call.func); - PyObject *kwnames; - RETURN_IF_FALSE(kwnames = PyTuple_New(nkwargs)); + VISIT(c, expr, p->v.MatchClass.cls); + PyObject *attr_names; + RETURN_IF_FALSE(attr_names = PyTuple_New(nattrs)); Py_ssize_t i; - for (i = 0; i < nkwargs; i++) { - PyObject *name = ((keyword_ty) asdl_seq_GET(kwargs, i))->arg; + for (i = 0; i < nattrs; i++) { + PyObject *name = asdl_seq_GET(kwd_attrs, i); Py_INCREF(name); - PyTuple_SET_ITEM(kwnames, i, name); + PyTuple_SET_ITEM(attr_names, i, name); } - ADDOP_LOAD_CONST_NEW(c, kwnames); + ADDOP_LOAD_CONST_NEW(c, attr_names); ADDOP_I(c, MATCH_CLASS, nargs); ADDOP_JUMP(c, POP_JUMP_IF_FALSE, fail_pop_1); NEXT_BLOCK(c); // TOS is now a tuple of (nargs + nkwargs) attributes. - for (i = 0; i < nargs + nkwargs; i++) { - expr_ty arg; + for (i = 0; i < nargs + nattrs; i++) { + pattern_ty pattern; if (i < nargs) { // Positional: - arg = asdl_seq_GET(args, i); + pattern = asdl_seq_GET(patterns, i); } else { // Keyword: - arg = ((keyword_ty) asdl_seq_GET(kwargs, i - nargs))->value; + pattern = asdl_seq_GET(kwd_patterns, i - nargs); } - if (WILDCARD_CHECK(arg)) { + if (WILDCARD_CHECK(pattern)) { continue; } // Get the i-th attribute, and match it against the i-th pattern: ADDOP(c, DUP_TOP); ADDOP_LOAD_CONST_NEW(c, PyLong_FromSsize_t(i)); ADDOP(c, BINARY_SUBSCR); - RETURN_IF_FALSE(compiler_pattern_subpattern(c, arg, pc)); + RETURN_IF_FALSE(compiler_pattern_subpattern(c, pattern, pc)); ADDOP_JUMP(c, POP_JUMP_IF_FALSE, fail_pop_1); NEXT_BLOCK(c); } @@ -5848,31 +5913,24 @@ compiler_pattern_class(struct compiler *c, expr_ty p, pattern_context *pc) return 1; } - -static int -compiler_pattern_literal(struct compiler *c, expr_ty p, pattern_context *pc) -{ - assert(p->kind == Constant_kind); - PyObject *v = p->v.Constant.value; - ADDOP_LOAD_CONST(c, v); - // Literal True, False, and None are compared by identity. All others use - // equality: - ADDOP_COMPARE(c, (v == Py_None || PyBool_Check(v)) ? Is : Eq); - return 1; -} - - static int -compiler_pattern_mapping(struct compiler *c, expr_ty p, pattern_context *pc) +compiler_pattern_mapping(struct compiler *c, pattern_ty p, pattern_context *pc) { + assert(p->kind == MatchMapping_kind); basicblock *end, *fail_pop_1, *fail_pop_3; RETURN_IF_FALSE(end = compiler_new_block(c)); RETURN_IF_FALSE(fail_pop_1 = compiler_new_block(c)); RETURN_IF_FALSE(fail_pop_3 = compiler_new_block(c)); - asdl_expr_seq *keys = p->v.Dict.keys; - asdl_expr_seq *values = p->v.Dict.values; - Py_ssize_t size = asdl_seq_LEN(values); - // A starred pattern will be a keyless value. It is guaranteed to be last: + asdl_expr_seq *keys = p->v.MatchMapping.keys; + asdl_pattern_seq *patterns = p->v.MatchMapping.patterns; + Py_ssize_t size = asdl_seq_LEN(keys); + Py_ssize_t npatterns = asdl_seq_LEN(patterns); + if (size != npatterns) { + // AST validator shouldn't let this happen, but if it does, + // just fail, don't crash out of the interpreter + const char * e = "keys (%d) / patterns (%d) length mismatch in mapping pattern"; + return compiler_error(c, e, size, npatterns); + } // A starred pattern will be a keyless value. It is guaranteed to be last: int star = size ? !asdl_seq_GET(keys, size - 1) : 0; ADDOP(c, MATCH_MAPPING); ADDOP_JUMP(c, POP_JUMP_IF_FALSE, fail_pop_1); @@ -5906,6 +5964,7 @@ compiler_pattern_mapping(struct compiler *c, expr_ty p, pattern_context *pc) if (key == NULL) { const char *e = "can't use starred name here " "(consider moving to end)"; + c->u->u_col_offset = ((pattern_ty) asdl_seq_GET(patterns, i))->col_offset; return compiler_error(c, e); } VISIT(c, expr, key); @@ -5917,14 +5976,14 @@ compiler_pattern_mapping(struct compiler *c, expr_ty p, pattern_context *pc) // So far so good. There's now a tuple of values on the stack to match // sub-patterns against: for (Py_ssize_t i = 0; i < size - star; i++) { - expr_ty value = asdl_seq_GET(values, i); - if (WILDCARD_CHECK(value)) { + pattern_ty pattern = asdl_seq_GET(patterns, i); + if (WILDCARD_CHECK(pattern)) { continue; } ADDOP(c, DUP_TOP); ADDOP_LOAD_CONST_NEW(c, PyLong_FromSsize_t(i)); ADDOP(c, BINARY_SUBSCR); - RETURN_IF_FALSE(compiler_pattern_subpattern(c, value, pc)); + RETURN_IF_FALSE(compiler_pattern_subpattern(c, pattern, pc)); ADDOP_JUMP(c, POP_JUMP_IF_FALSE, fail_pop_3); NEXT_BLOCK(c); } @@ -5933,7 +5992,7 @@ compiler_pattern_mapping(struct compiler *c, expr_ty p, pattern_context *pc) if (star) { // If we had a starred name, bind a dict of remaining items to it: ADDOP(c, COPY_DICT_WITHOUT_KEYS); - PyObject *id = asdl_seq_GET(values, size - 1)->v.Name.id; + identifier id = asdl_seq_GET(patterns, size - 1)->v.MatchAs.target; RETURN_IF_FALSE(pattern_helper_store_name(c, id, pc)); } else { @@ -5957,9 +6016,8 @@ compiler_pattern_mapping(struct compiler *c, expr_ty p, pattern_context *pc) return 1; } - static int -compiler_pattern_or(struct compiler *c, expr_ty p, pattern_context *pc) +compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchOr_kind); // control is the set of names bound by the first alternative. If all of the @@ -5975,7 +6033,7 @@ compiler_pattern_or(struct compiler *c, expr_ty p, pattern_context *pc) int allow_irrefutable = pc->allow_irrefutable; for (Py_ssize_t i = 0; i < size; i++) { // NOTE: Can't use our nice returning macros in here: they'll leak sets! - expr_ty alt = asdl_seq_GET(p->v.MatchOr.patterns, i); + pattern_ty alt = asdl_seq_GET(p->v.MatchOr.patterns, i); pc->stores = PySet_New(stores_init); // An irrefutable sub-pattern must be last, if it is allowed at all: int is_last = i == size - 1; @@ -6031,28 +6089,28 @@ compiler_pattern_or(struct compiler *c, expr_ty p, pattern_context *pc) static int -compiler_pattern_sequence(struct compiler *c, expr_ty p, pattern_context *pc) +compiler_pattern_sequence(struct compiler *c, pattern_ty p, pattern_context *pc) { - assert(p->kind == List_kind || p->kind == Tuple_kind); - asdl_expr_seq *values = (p->kind == Tuple_kind) ? p->v.Tuple.elts - : p->v.List.elts; - Py_ssize_t size = asdl_seq_LEN(values); + assert(p->kind == MatchSequence_kind); + asdl_pattern_seq *patterns = p->v.MatchSequence.patterns; + Py_ssize_t size = asdl_seq_LEN(patterns); Py_ssize_t star = -1; int only_wildcard = 1; int star_wildcard = 0; // Find a starred name, if it exists. There may be at most one: for (Py_ssize_t i = 0; i < size; i++) { - expr_ty value = asdl_seq_GET(values, i); - if (value->kind == Starred_kind) { - value = value->v.Starred.value; + pattern_ty pattern = asdl_seq_GET(patterns, i); + if (pattern->kind == MatchStar_kind) { if (star >= 0) { const char *e = "multiple starred names in sequence pattern"; return compiler_error(c, e); } - star_wildcard = WILDCARD_CHECK(value); + star_wildcard = WILDCARD_STAR_CHECK(pattern); + only_wildcard &= star_wildcard; star = i; + continue; } - only_wildcard &= WILDCARD_CHECK(value); + only_wildcard &= WILDCARD_CHECK(pattern); } basicblock *end, *fail_pop_1; RETURN_IF_FALSE(end = compiler_new_block(c)); @@ -6082,10 +6140,10 @@ compiler_pattern_sequence(struct compiler *c, expr_ty p, pattern_context *pc) ADDOP_LOAD_CONST(c, Py_True); } else if (star_wildcard) { - RETURN_IF_FALSE(pattern_helper_sequence_subscr(c, values, star, pc)); + RETURN_IF_FALSE(pattern_helper_sequence_subscr(c, patterns, star, pc)); } else { - RETURN_IF_FALSE(pattern_helper_sequence_unpack(c, values, star, pc)); + RETURN_IF_FALSE(pattern_helper_sequence_unpack(c, patterns, star, pc)); } ADDOP_JUMP(c, JUMP_FORWARD, end); compiler_use_next_block(c, fail_pop_1); @@ -6095,24 +6153,10 @@ compiler_pattern_sequence(struct compiler *c, expr_ty p, pattern_context *pc) return 1; } - static int -compiler_pattern_value(struct compiler *c, expr_ty p, pattern_context *pc) +compiler_pattern_wildcard(struct compiler *c, pattern_ty p, pattern_context *pc) { - assert(p->kind == Attribute_kind); - assert(p->v.Attribute.ctx == Load); - VISIT(c, expr, p); - ADDOP_COMPARE(c, Eq); - return 1; -} - - -static int -compiler_pattern_wildcard(struct compiler *c, expr_ty p, pattern_context *pc) -{ - assert(p->kind == Name_kind); - assert(p->v.Name.ctx == Store); - assert(WILDCARD_CHECK(p)); + assert(p->kind == MatchAlways_kind); if (!pc->allow_irrefutable) { // Whoops, can't have a wildcard here! const char *e = "wildcard makes remaining patterns unreachable"; @@ -6123,44 +6167,54 @@ compiler_pattern_wildcard(struct compiler *c, expr_ty p, pattern_context *pc) return 1; } +static int +compiler_pattern_value(struct compiler *c, pattern_ty p, pattern_context *pc) +{ + assert(p->kind == MatchValue_kind); + VISIT(c, expr, p->v.MatchValue.value); + ADDOP_COMPARE(c, Eq); + return 1; +} static int -compiler_pattern(struct compiler *c, expr_ty p, pattern_context *pc) +compiler_pattern_constant(struct compiler *c, pattern_ty p, pattern_context *pc) +{ + assert(p->kind == MatchConstant_kind); + ADDOP_LOAD_CONST(c, p->v.MatchConstant.value); + ADDOP_COMPARE(c, Is); + return 1; +} + +static int +compiler_pattern(struct compiler *c, pattern_ty p, pattern_context *pc) { SET_LOC(c, p); switch (p->kind) { - case Attribute_kind: + case MatchAlways_kind: + return compiler_pattern_wildcard(c, p, pc); + case MatchValue_kind: return compiler_pattern_value(c, p, pc); - case BinOp_kind: - // Because we allow "2+2j", things like "2+2" make it this far: - return compiler_error(c, "patterns cannot include operators"); - case Call_kind: - return compiler_pattern_class(c, p, pc); - case Constant_kind: - return compiler_pattern_literal(c, p, pc); - case Dict_kind: - return compiler_pattern_mapping(c, p, pc); - case JoinedStr_kind: - // Because we allow strings, f-strings make it this far: - return compiler_error(c, "patterns cannot include f-strings"); - case List_kind: - case Tuple_kind: + case MatchConstant_kind: + return compiler_pattern_constant(c, p, pc); + case MatchSequence_kind: return compiler_pattern_sequence(c, p, pc); + case MatchMapping_kind: + return compiler_pattern_mapping(c, p, pc); + case MatchClass_kind: + return compiler_pattern_class(c, p, pc); + case MatchStar_kind: + return compiler_pattern_star(c, p, pc); case MatchAs_kind: return compiler_pattern_as(c, p, pc); case MatchOr_kind: return compiler_pattern_or(c, p, pc); - case Name_kind: - if (WILDCARD_CHECK(p)) { - return compiler_pattern_wildcard(c, p, pc); - } - return compiler_pattern_capture(c, p, pc); - default: - Py_UNREACHABLE(); } + // AST validator shouldn't let this happen, but if it does, + // just fail, don't crash out of the interpreter + const char *e = "invalid match pattern node in AST (kind=%d)"; + return compiler_error(c, e, p->kind); } - static int compiler_match(struct compiler *c, stmt_ty s) { @@ -6168,7 +6222,7 @@ compiler_match(struct compiler *c, stmt_ty s) basicblock *next, *end; RETURN_IF_FALSE(end = compiler_new_block(c)); Py_ssize_t cases = asdl_seq_LEN(s->v.Match.cases); - assert(cases); + assert(cases > 0); pattern_context pc; // We use pc.stores to track: // - Repeated name assignments in the same pattern. @@ -6222,10 +6276,8 @@ compiler_match(struct compiler *c, stmt_ty s) return 1; } - #undef WILDCARD_CHECK - /* End of the compiler section, beginning of the assembler section */ /* do depth-first search of basic block graph, starting with block. From 2a319fd36f0109d1f707453566dbb41bfe62e7a8 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Thu, 22 Apr 2021 23:36:48 +1000 Subject: [PATCH 03/38] Link to the validator ticket --- Python/ast.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ast.c b/Python/ast.c index f07431660b0c19..5f32c46fbbe21e 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -319,7 +319,7 @@ validate_expr(expr_ty exp, expr_context_ty ctx) static int validate_pattern(pattern_ty p) { - // Coming soon (thanks Batuhan)! + // Coming soon: https://bugs.python.org/issue43897 (thanks Batuhan)! // TODO: Potentially ensure no subnodes use "_" as an ordinary identifier switch (p->kind) { case MatchAlways_kind: From 14a5b2b69babea8712e2ac2a56bacc16b49d74ea Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Thu, 22 Apr 2021 23:41:02 +1000 Subject: [PATCH 04/38] Ensure unparsing tests always cover pattern matching --- Lib/test/test_unparse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index ce03272ad30b2d..9f67b49f3a6b2b 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -518,7 +518,8 @@ class DirectoryTestCase(ASTTestCase): lib_dir = pathlib.Path(__file__).parent / ".." test_directories = (lib_dir, lib_dir / "test") run_always_files = {"test_grammar.py", "test_syntax.py", "test_compile.py", - "test_ast.py", "test_asdl_parser.py", "test_fstring.py"} + "test_ast.py", "test_asdl_parser.py", "test_fstring.py", + "test_patma.py"} _files_to_test = None From ba2932fc070e7623e96be0df6b75b98cc704e603 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Fri, 23 Apr 2021 18:57:46 +1000 Subject: [PATCH 05/38] Get symtable.c compiling --- Include/internal/pycore_symtable.h | 1 - Python/symtable.c | 70 +++++++++++++++++++++++------- 2 files changed, 54 insertions(+), 17 deletions(-) diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 5d34a6c951cab3..86882d834327df 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -32,7 +32,6 @@ struct symtable { the symbol table */ int recursion_depth; /* current recursion depth */ int recursion_limit; /* recursion limit */ - int in_pattern; /* whether we are currently in a pattern */ }; typedef struct _symtable_entry { diff --git a/Python/symtable.c b/Python/symtable.c index d148a563a3eb5a..1fdac53b909e94 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -210,6 +210,7 @@ static int symtable_implicit_arg(struct symtable *st, int pos); static int symtable_visit_annotations(struct symtable *st, arguments_ty, expr_ty); static int symtable_visit_withitem(struct symtable *st, withitem_ty item); static int symtable_visit_match_case(struct symtable *st, match_case_ty m); +static int symtable_visit_pattern(struct symtable *st, pattern_ty s); static identifier top = NULL, lambda = NULL, genexpr = NULL, @@ -242,7 +243,6 @@ symtable_new(void) goto fail; st->st_cur = NULL; st->st_private = NULL; - st->in_pattern = 0; return st; fail: _PySymtable_Free(st); @@ -1644,13 +1644,6 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT(st, expr, e->v.Slice.step) break; case Name_kind: - // Don't make "_" a local when used in a pattern: - if (st->in_pattern && - e->v.Name.ctx == Store && - _PyUnicode_EqualToASCIIString(e->v.Name.id, "_")) - { - break; - } if (!symtable_add_def(st, e->v.Name.id, e->v.Name.ctx == Load ? USE : DEF_LOCAL)) VISIT_QUIT(st, 0); @@ -1670,12 +1663,61 @@ symtable_visit_expr(struct symtable *st, expr_ty e) case Tuple_kind: VISIT_SEQ(st, expr, e->v.Tuple.elts); break; + } + VISIT_QUIT(st, 1); +} + +static int +symtable_visit_pattern(struct symtable *st, pattern_ty p) +{ + if (++st->recursion_depth > st->recursion_limit) { + PyErr_SetString(PyExc_RecursionError, + "maximum recursion depth exceeded during compilation"); + VISIT_QUIT(st, 0); + } + switch (p->kind) { + case MatchAlways_kind: + /* Nothing to do here. */ + break; + case MatchValue_kind: + VISIT(st, expr, p->v.MatchValue.value); + break; + case MatchConstant_kind: + /* Nothing to do here. */ + break; + case MatchSequence_kind: + VISIT_SEQ(st, pattern, p->v.MatchSequence.patterns); + break; + case MatchStar_kind: + { + PyObject *target_name = p->v.MatchStar.target; + if (target_name) { + symtable_add_def(st, target_name, DEF_LOCAL); + } + } break; + case MatchMapping_kind: + if (p->v.MatchMapping.keys) { + VISIT_SEQ_WITH_NULL(st, expr, p->v.MatchMapping.keys); + VISIT_SEQ(st, pattern, p->v.MatchMapping.patterns); + } + break; + case MatchClass_kind: + VISIT(st, expr, p->v.MatchClass.cls); + if (p->v.MatchClass.patterns) { + VISIT_SEQ(st, pattern, p->v.MatchClass.patterns); + } + if (p->v.MatchClass.kwd_patterns) { + VISIT_SEQ(st, pattern, p->v.MatchClass.kwd_patterns); + } + break; case MatchAs_kind: - VISIT(st, expr, e->v.MatchAs.pattern); - symtable_add_def(st, e->v.MatchAs.name, DEF_LOCAL); + if (p->v.MatchAs.pattern) { + VISIT(st, pattern, p->v.MatchAs.pattern); + } + symtable_add_def(st, p->v.MatchAs.target, DEF_LOCAL); break; case MatchOr_kind: - VISIT_SEQ(st, expr, e->v.MatchOr.patterns); + VISIT_SEQ(st, pattern, p->v.MatchOr.patterns); break; } VISIT_QUIT(st, 1); @@ -1798,11 +1840,7 @@ symtable_visit_withitem(struct symtable *st, withitem_ty item) static int symtable_visit_match_case(struct symtable *st, match_case_ty m) { - assert(!st->in_pattern); - st->in_pattern = 1; - VISIT(st, expr, m->pattern); - assert(st->in_pattern); - st->in_pattern = 0; + VISIT(st, pattern, m->pattern); if (m->guard) { VISIT(st, expr, m->guard); } From 1dccb6fa0c6a8e25446d5f507612a3c8648ea077 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Fri, 23 Apr 2021 19:03:52 +1000 Subject: [PATCH 06/38] Require end attributes on pattern nodes --- Parser/Python.asdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parser/Python.asdl b/Parser/Python.asdl index cac812dc3f6d90..0d024b3be73eeb 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -138,7 +138,7 @@ module Python | MatchAs(pattern? pattern, identifier target) | MatchOr(pattern* patterns) - attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) + attributes (int lineno, int col_offset, int end_lineno, int end_col_offset) type_ignore = TypeIgnore(int lineno, string tag) } From 207a43e3e2decb6e834b1003fe22e90a0225ffe5 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Fri, 23 Apr 2021 19:14:46 +1000 Subject: [PATCH 07/38] MatchConstant -> MatchSingleton --- Grammar/python.gram | 6 +-- Include/internal/pycore_ast.h | 9 +++-- Include/internal/pycore_ast_state.h | 2 +- Parser/Python.asdl | 2 +- Parser/parser.c | 6 +-- Python/Python-ast.c | 61 +++++++++++++---------------- Python/ast.c | 4 +- Python/ast_opt.c | 2 +- Python/compile.c | 6 +-- Python/symtable.c | 2 +- 10 files changed, 48 insertions(+), 52 deletions(-) diff --git a/Grammar/python.gram b/Grammar/python.gram index a194c367961e2f..8de06a1da86a3f 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -262,9 +262,9 @@ literal_pattern[pattern_ty]: | value=signed_number !('+' | '-') { _PyAST_MatchValue(value, EXTRA) } | value=complex_number { _PyAST_MatchValue(value, EXTRA) } | value=strings { _PyAST_MatchValue(value, EXTRA) } - | 'None' { _PyAST_MatchConstant(Py_None, EXTRA) } - | 'True' { _PyAST_MatchConstant(Py_True, EXTRA) } - | 'False' { _PyAST_MatchConstant(Py_False, EXTRA) } + | 'None' { _PyAST_MatchSingleton(Py_None, EXTRA) } + | 'True' { _PyAST_MatchSingleton(Py_True, EXTRA) } + | 'False' { _PyAST_MatchSingleton(Py_False, EXTRA) } # Literal expressions are used to restrict permitted mapping pattern keys literal_expr[expr_ty]: diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index 7726aa6ab7017c..e385730f7dc8d5 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -560,7 +560,7 @@ struct _match_case { }; enum _pattern_kind {MatchAlways_kind=1, MatchValue_kind=2, - MatchConstant_kind=3, MatchSequence_kind=4, + MatchSingleton_kind=3, MatchSequence_kind=4, MatchMapping_kind=5, MatchClass_kind=6, MatchStar_kind=7, MatchAs_kind=8, MatchOr_kind=9}; struct _pattern { @@ -572,7 +572,7 @@ struct _pattern { struct { constant value; - } MatchConstant; + } MatchSingleton; struct { asdl_pattern_seq *patterns; @@ -811,8 +811,9 @@ pattern_ty _PyAST_MatchAlways(int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); pattern_ty _PyAST_MatchValue(expr_ty value, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); -pattern_ty _PyAST_MatchConstant(constant value, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); +pattern_ty _PyAST_MatchSingleton(constant value, int lineno, int col_offset, + int end_lineno, int end_col_offset, PyArena + *arena); pattern_ty _PyAST_MatchSequence(asdl_pattern_seq * patterns, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index 4bdde03c029fd6..b7094d73885b75 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -94,10 +94,10 @@ struct ast_state { PyObject *MatchAlways_type; PyObject *MatchAs_type; PyObject *MatchClass_type; - PyObject *MatchConstant_type; PyObject *MatchMapping_type; PyObject *MatchOr_type; PyObject *MatchSequence_type; + PyObject *MatchSingleton_type; PyObject *MatchStar_type; PyObject *MatchValue_type; PyObject *Match_type; diff --git a/Parser/Python.asdl b/Parser/Python.asdl index 0d024b3be73eeb..d0ce6b25dd1716 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -128,7 +128,7 @@ module Python pattern = MatchAlways | MatchValue(expr value) - | MatchConstant(constant value) + | MatchSingleton(constant value) | MatchSequence(pattern* patterns) | MatchMapping(expr* keys, pattern* patterns) | MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns) diff --git a/Parser/parser.c b/Parser/parser.c index 3a10f82e8b7f2b..1556f02b3c034a 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -6040,7 +6040,7 @@ literal_pattern_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_MatchConstant ( Py_None , EXTRA ); + _res = _PyAST_MatchSingleton ( Py_None , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -6073,7 +6073,7 @@ literal_pattern_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_MatchConstant ( Py_True , EXTRA ); + _res = _PyAST_MatchSingleton ( Py_True , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -6106,7 +6106,7 @@ literal_pattern_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_MatchConstant ( Py_False , EXTRA ); + _res = _PyAST_MatchSingleton ( Py_False , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); diff --git a/Python/Python-ast.c b/Python/Python-ast.c index ea63245cf3bdb2..d21cd946fdaa4b 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -108,10 +108,10 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(state->MatchAlways_type); Py_CLEAR(state->MatchAs_type); Py_CLEAR(state->MatchClass_type); - Py_CLEAR(state->MatchConstant_type); Py_CLEAR(state->MatchMapping_type); Py_CLEAR(state->MatchOr_type); Py_CLEAR(state->MatchSequence_type); + Py_CLEAR(state->MatchSingleton_type); Py_CLEAR(state->MatchStar_type); Py_CLEAR(state->MatchValue_type); Py_CLEAR(state->Match_type); @@ -714,7 +714,7 @@ static PyObject* ast2obj_pattern(struct ast_state *state, void*); static const char * const MatchValue_fields[]={ "value", }; -static const char * const MatchConstant_fields[]={ +static const char * const MatchSingleton_fields[]={ "value", }; static const char * const MatchSequence_fields[]={ @@ -1774,7 +1774,7 @@ init_types(struct ast_state *state) state->pattern_type = make_type(state, "pattern", state->AST_type, NULL, 0, "pattern = MatchAlways\n" " | MatchValue(expr value)\n" - " | MatchConstant(constant value)\n" + " | MatchSingleton(constant value)\n" " | MatchSequence(pattern* patterns)\n" " | MatchMapping(expr* keys, pattern* patterns)\n" " | MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns)\n" @@ -1784,11 +1784,6 @@ init_types(struct ast_state *state) if (!state->pattern_type) return 0; if (!add_attributes(state, state->pattern_type, pattern_attributes, 4)) return 0; - if (PyObject_SetAttr(state->pattern_type, state->end_lineno, Py_None) == -1) - return 0; - if (PyObject_SetAttr(state->pattern_type, state->end_col_offset, Py_None) - == -1) - return 0; state->MatchAlways_type = make_type(state, "MatchAlways", state->pattern_type, NULL, 0, "MatchAlways"); @@ -1798,11 +1793,11 @@ init_types(struct ast_state *state) 1, "MatchValue(expr value)"); if (!state->MatchValue_type) return 0; - state->MatchConstant_type = make_type(state, "MatchConstant", - state->pattern_type, - MatchConstant_fields, 1, - "MatchConstant(constant value)"); - if (!state->MatchConstant_type) return 0; + state->MatchSingleton_type = make_type(state, "MatchSingleton", + state->pattern_type, + MatchSingleton_fields, 1, + "MatchSingleton(constant value)"); + if (!state->MatchSingleton_type) return 0; state->MatchSequence_type = make_type(state, "MatchSequence", state->pattern_type, MatchSequence_fields, 1, @@ -3433,20 +3428,20 @@ _PyAST_MatchValue(expr_ty value, int lineno, int col_offset, int end_lineno, } pattern_ty -_PyAST_MatchConstant(constant value, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena) +_PyAST_MatchSingleton(constant value, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) { pattern_ty p; if (!value) { PyErr_SetString(PyExc_ValueError, - "field 'value' is required for MatchConstant"); + "field 'value' is required for MatchSingleton"); return NULL; } p = (pattern_ty)_PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; - p->kind = MatchConstant_kind; - p->v.MatchConstant.value = value; + p->kind = MatchSingleton_kind; + p->v.MatchSingleton.value = value; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -5190,11 +5185,11 @@ ast2obj_pattern(struct ast_state *state, void* _o) goto failed; Py_DECREF(value); break; - case MatchConstant_kind: - tp = (PyTypeObject *)state->MatchConstant_type; + case MatchSingleton_kind: + tp = (PyTypeObject *)state->MatchSingleton_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_constant(state, o->v.MatchConstant.value); + value = ast2obj_constant(state, o->v.MatchSingleton.value); if (!value) goto failed; if (PyObject_SetAttr(result, state->value, value) == -1) goto failed; @@ -10290,9 +10285,9 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (_PyObject_LookupAttr(obj, state->end_lineno, &tmp) < 0) { return 1; } - if (tmp == NULL || tmp == Py_None) { - Py_CLEAR(tmp); - end_lineno = 0; + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"end_lineno\" missing from pattern"); + return 1; } else { int res; @@ -10303,9 +10298,9 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (_PyObject_LookupAttr(obj, state->end_col_offset, &tmp) < 0) { return 1; } - if (tmp == NULL || tmp == Py_None) { - Py_CLEAR(tmp); - end_col_offset = 0; + if (tmp == NULL) { + PyErr_SetString(PyExc_TypeError, "required field \"end_col_offset\" missing from pattern"); + return 1; } else { int res; @@ -10351,7 +10346,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (*out == NULL) goto failed; return 0; } - tp = state->MatchConstant_type; + tp = state->MatchSingleton_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { return 1; @@ -10363,7 +10358,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, return 1; } if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from MatchConstant"); + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from MatchSingleton"); return 1; } else { @@ -10372,8 +10367,8 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = _PyAST_MatchConstant(value, lineno, col_offset, end_lineno, - end_col_offset, arena); + *out = _PyAST_MatchSingleton(value, lineno, col_offset, end_lineno, + end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -11165,8 +11160,8 @@ astmodule_exec(PyObject *m) if (PyModule_AddObjectRef(m, "MatchValue", state->MatchValue_type) < 0) { return -1; } - if (PyModule_AddObjectRef(m, "MatchConstant", state->MatchConstant_type) < - 0) { + if (PyModule_AddObjectRef(m, "MatchSingleton", state->MatchSingleton_type) + < 0) { return -1; } if (PyModule_AddObjectRef(m, "MatchSequence", state->MatchSequence_type) < diff --git a/Python/ast.c b/Python/ast.c index 5f32c46fbbe21e..35b3490a9a0821 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -330,9 +330,9 @@ validate_pattern(pattern_ty p) // (will need to allow selected unary ops & binops if // validation is performed before constant folding...) return validate_expr(p->v.MatchValue.value, Load); - case MatchConstant_kind: + case MatchSingleton_kind: // TODO: Check constant is specifically None, True, or False - return validate_constant(p->v.MatchConstant.value); + return validate_constant(p->v.MatchSingleton.value); case MatchSequence_kind: // TODO: Validate all subpatterns // return validate_patterns(p->v.MatchSequence.patterns); diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 085ebd6fcc2464..17e6d4ed297f91 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -790,7 +790,7 @@ astfold_pattern(pattern_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) case MatchValue_kind: CALL(astfold_expr, expr_ty, node_->v.MatchValue.value); break; - case MatchConstant_kind: + case MatchSingleton_kind: break; case MatchSequence_kind: CALL_SEQ(astfold_pattern, pattern, node_->v.MatchSequence.patterns); diff --git a/Python/compile.c b/Python/compile.c index 9eb4e901b9c0cc..a3205af25b4f6f 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -6179,8 +6179,8 @@ compiler_pattern_value(struct compiler *c, pattern_ty p, pattern_context *pc) static int compiler_pattern_constant(struct compiler *c, pattern_ty p, pattern_context *pc) { - assert(p->kind == MatchConstant_kind); - ADDOP_LOAD_CONST(c, p->v.MatchConstant.value); + assert(p->kind == MatchSingleton_kind); + ADDOP_LOAD_CONST(c, p->v.MatchSingleton.value); ADDOP_COMPARE(c, Is); return 1; } @@ -6194,7 +6194,7 @@ compiler_pattern(struct compiler *c, pattern_ty p, pattern_context *pc) return compiler_pattern_wildcard(c, p, pc); case MatchValue_kind: return compiler_pattern_value(c, p, pc); - case MatchConstant_kind: + case MatchSingleton_kind: return compiler_pattern_constant(c, p, pc); case MatchSequence_kind: return compiler_pattern_sequence(c, p, pc); diff --git a/Python/symtable.c b/Python/symtable.c index 1fdac53b909e94..4dc2001ad8b4aa 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1682,7 +1682,7 @@ symtable_visit_pattern(struct symtable *st, pattern_ty p) case MatchValue_kind: VISIT(st, expr, p->v.MatchValue.value); break; - case MatchConstant_kind: + case MatchSingleton_kind: /* Nothing to do here. */ break; case MatchSequence_kind: From ba104d1dc01fe15ea52a2d5602327f58c9e2cea0 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Fri, 23 Apr 2021 19:21:24 +1000 Subject: [PATCH 08/38] Formatting tweak --- Python/symtable.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/symtable.c b/Python/symtable.c index 4dc2001ad8b4aa..1661e2cdbfafa5 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1694,7 +1694,8 @@ symtable_visit_pattern(struct symtable *st, pattern_ty p) if (target_name) { symtable_add_def(st, target_name, DEF_LOCAL); } - } break; + } + break; case MatchMapping_kind: if (p->v.MatchMapping.keys) { VISIT_SEQ_WITH_NULL(st, expr, p->v.MatchMapping.keys); From 7cdb7355224a7a9d1f8b2caafc1b56309eab6967 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Fri, 23 Apr 2021 19:47:51 +1000 Subject: [PATCH 09/38] Fix segfault in wildcard name bindings --- Lib/test/test_patma.py | 2 ++ Python/ast.c | 5 ++++- Python/ast_opt.c | 4 +++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py index 40580be86b0848..51895199e7ff65 100644 --- a/Lib/test/test_patma.py +++ b/Lib/test/test_patma.py @@ -2885,6 +2885,8 @@ def run_perf(self): @staticmethod def setUpClass(): raise unittest.SkipTest("performance testing") +''' # Disable compilation +''' # End disable compilation """ diff --git a/Python/ast.c b/Python/ast.c index 35b3490a9a0821..a0523baf99a4fb 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -369,7 +369,10 @@ validate_pattern(pattern_ty p) // Nothing to check (except to potentially block "_" as an identifer) break; case MatchAs_kind: - return validate_pattern(p->v.MatchAs.pattern); + if (p->v.MatchAs.pattern) { + return validate_pattern(p->v.MatchAs.pattern); + } + return 1; case MatchOr_kind: // TODO: Validate all subpatterns // return validate_patterns(p->v.MatchOr.patterns); diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 17e6d4ed297f91..e071ccb2130e9a 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -807,7 +807,9 @@ astfold_pattern(pattern_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) case MatchStar_kind: break; case MatchAs_kind: - CALL(astfold_pattern, expr_ty, node_->v.MatchAs.pattern); + if (node_->v.MatchAs.pattern) { + CALL(astfold_pattern, expr_ty, node_->v.MatchAs.pattern); + } break; case MatchOr_kind: CALL_SEQ(astfold_pattern, pattern, node_->v.MatchOr.patterns); From efc0f73706ce681c1628001d2c4673e163b352de Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Fri, 23 Apr 2021 19:50:22 +1000 Subject: [PATCH 10/38] Get test_patma compiling --- Python/ast.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index a0523baf99a4fb..052a1a2b07fe7f 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -352,9 +352,9 @@ validate_pattern(pattern_ty p) // return validate_patterns(p->v.MatchMapping.patterns); return 1; case MatchClass_kind: - if (asdl_seq_LEN(p->v.MatchMapping.keys) != asdl_seq_LEN(p->v.MatchMapping.patterns)) { + if (asdl_seq_LEN(p->v.MatchClass.kwd_attrs) != asdl_seq_LEN(p->v.MatchClass.kwd_patterns)) { PyErr_SetString(PyExc_ValueError, - "MatchMapping doesn't have the same number of keys as patterns"); + "MatchClass doesn't have the same number of keyword attributes as patterns"); return 0; } // TODO: Restrict cls lookup to being a name or attribute From a0e13dae2ca0e7183b0b3242d1e08afa2dfc3465 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Fri, 23 Apr 2021 22:03:16 +1000 Subject: [PATCH 11/38] Reject matching 0+0 and f-strings --- Lib/test/test_patma.py | 22 +++++++- Python/ast.c | 112 +++++++++++++++++++++++++++++++++++++++-- Python/compile.c | 15 +++++- 3 files changed, 142 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py index 51895199e7ff65..68f8ba8c0daf0a 100644 --- a/Lib/test/test_patma.py +++ b/Lib/test/test_patma.py @@ -2429,6 +2429,17 @@ def test_patma_240(self): pass """) + @no_perf + def _disabled_test_patma_240b(self): + # Test disabled until bpo-43897 fleshes out the AST validator + # (the initial skeleton doesn't recurse into mapping patterns) + # Ensure mapping keys are also restricted + self.assert_syntax_error(""" + match ...: + case {0+0:_}: + pass + """) + @no_perf def test_patma_241(self): self.assert_syntax_error(""" @@ -2437,6 +2448,15 @@ def test_patma_241(self): pass """) + @no_perf + def test_patma_241b(self): + # Ensure mapping keys are also restricted + self.assert_syntax_error(""" + match ...: + case {f"":_}: + pass + """) + @no_perf def test_patma_242(self): self.assert_syntax_error(""" @@ -2885,8 +2905,6 @@ def run_perf(self): @staticmethod def setUpClass(): raise unittest.SkipTest("performance testing") -''' # Disable compilation -''' # End disable compilation """ diff --git a/Python/ast.c b/Python/ast.c index 052a1a2b07fe7f..39d0d01d616bb0 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -6,6 +6,7 @@ #include "pycore_ast.h" // asdl_stmt_seq #include +#include static int validate_stmts(asdl_stmt_seq *); static int validate_exprs(asdl_expr_seq*, expr_context_ty, int); @@ -316,19 +317,122 @@ validate_expr(expr_ty exp, expr_context_ty ctx) return 0; } +static int +ensure_literal_number(expr_ty exp, bool allow_real, bool allow_imaginary) +{ + assert(exp->kind == Constant_kind); + PyObject *value = exp->v.Constant.value; + return (allow_real && PyFloat_CheckExact(value)) || + (allow_real && PyLong_CheckExact(value)) || + (allow_imaginary && PyComplex_CheckExact(value)); +} + +static int +ensure_literal_negative(expr_ty exp, bool allow_real, bool allow_imaginary) +{ + assert(exp->kind == UnaryOp_kind); + // Must be negation ... + if (exp->v.UnaryOp.op != USub) { + return 0; + } + // ... of a constant ... + expr_ty operand = exp->v.UnaryOp.operand; + if (operand->kind != Constant_kind) { + return 0; + } + // ... number + return ensure_literal_number(operand, allow_real, allow_imaginary); +} + +static int +ensure_literal_complex(expr_ty exp) +{ + assert(exp->kind == BinOp_kind); + expr_ty left = exp->v.BinOp.left; + expr_ty right = exp->v.BinOp.right; + // Ensure op is addition or subtraction + if (exp->v.BinOp.op != Add && exp->v.BinOp.op != Sub) { + return 0; + } + // Check LHS is a real number + switch (left->kind) + { + case Constant_kind: + if (!ensure_literal_number(left, /*real=*/true, /*imaginary=*/false)) { + return 0; + } + break; + case UnaryOp_kind: + if (!ensure_literal_negative(left, /*real=*/true, /*imaginary=*/false)) { + return 0; + } + break; + default: + return 0; + } + // Check RHS is an imaginary number + switch (right->kind) + { + case Constant_kind: + if (!ensure_literal_number(right, /*real=*/false, /*imaginary=*/true)) { + return 0; + } + break; + case UnaryOp_kind: + if (!ensure_literal_negative(right, /*real=*/false, /*imaginary=*/true)) { + return 0; + } + break; + default: + return 0; + } + return 1; +} + +static int +validate_pattern_match_value(expr_ty exp) +{ + switch (exp->kind) + { + case Constant_kind: + case Attribute_kind: + // Constants and attribute lookups are always permitted + return 1; + case UnaryOp_kind: + // Negated numbers are permitted (whether real or imaginary) + // Compiler will complain if AST folding doesn't create a constant + if (ensure_literal_negative(exp, /*real=*/true, /*imaginary=*/true)) { + return 1; + } + break; + case BinOp_kind: + // Complex literals are permitted + // Compiler will complain if AST folding doesn't create a constant + if (ensure_literal_complex(exp)) { + return 1; + } + break; + default: + break; + } + PyErr_SetString(PyExc_SyntaxError, + "patterns may only match literals and attribute lookups"); + return 0; +} + static int validate_pattern(pattern_ty p) { // Coming soon: https://bugs.python.org/issue43897 (thanks Batuhan)! - // TODO: Potentially ensure no subnodes use "_" as an ordinary identifier + // TODO: Ensure no subnodes use "_" as an ordinary identifier switch (p->kind) { case MatchAlways_kind: // Nothing to check return 1; case MatchValue_kind: - // TODO: Check value is a constant or an attribute lookup - // (will need to allow selected unary ops & binops if - // validation is performed before constant folding...) + if (!validate_pattern_match_value(p->v.MatchValue.value)) { + return 0; + } return validate_expr(p->v.MatchValue.value, Load); case MatchSingleton_kind: // TODO: Check constant is specifically None, True, or False diff --git a/Python/compile.c b/Python/compile.c index a3205af25b4f6f..5654a6642160e6 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -5589,6 +5589,10 @@ compiler_slice(struct compiler *c, expr_ty s) #define WILDCARD_STAR_CHECK(N) \ ((N)->kind == MatchStar_kind && !(N)->v.MatchStar.target) +// Limit permitted subexpressions, even if the parser & AST validator let them through +#define MATCH_VALUE_EXPR(N) \ + ((N)->kind == Constant_kind || (N)->kind == Attribute_kind) + static int pattern_helper_store_name(struct compiler *c, identifier n, pattern_context *pc) { @@ -5967,6 +5971,10 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, pattern_context *pc) c->u->u_col_offset = ((pattern_ty) asdl_seq_GET(patterns, i))->col_offset; return compiler_error(c, e); } + if (!MATCH_VALUE_EXPR(key)) { + const char *e = "mapping pattern keys may only match literals and attribute lookups"; + return compiler_error(c, e); + } VISIT(c, expr, key); } ADDOP_I(c, BUILD_TUPLE, size - star); @@ -6171,7 +6179,12 @@ static int compiler_pattern_value(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchValue_kind); - VISIT(c, expr, p->v.MatchValue.value); + expr_ty value = p->v.MatchValue.value; + if (!MATCH_VALUE_EXPR(value)) { + const char *e = "patterns may only match literals and attribute lookups"; + return compiler_error(c, e); + } + VISIT(c, expr, value); ADDOP_COMPARE(c, Eq); return 1; } From 54940a3817df3046da3f9c51d4d426a73b2ec786 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Fri, 23 Apr 2021 23:06:33 +1000 Subject: [PATCH 12/38] Dedicated MatchMapping arg for storing remaining keys --- Grammar/python.gram | 21 ++- Include/internal/pycore_ast.h | 6 +- Include/internal/pycore_ast_state.h | 1 + Parser/Python.asdl | 4 +- Parser/parser.c | 213 ++++++++++++++++++++-------- Python/Python-ast.c | 39 ++++- Python/ast.c | 8 +- Python/compile.c | 30 ++-- Python/symtable.c | 10 +- 9 files changed, 233 insertions(+), 99 deletions(-) diff --git a/Grammar/python.gram b/Grammar/python.gram index 8de06a1da86a3f..9819b0b3960341 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -323,20 +323,29 @@ star_pattern[pattern_ty]: _PyAST_MatchStar(NULL, EXTRA) } mapping_pattern[pattern_ty]: - | '{' items=items_pattern? '}' { + | '{' '}' { + _PyAST_MatchMapping(NULL, NULL, NULL, EXTRA) } + | '{' rest=double_star_pattern ','? '}' { + _PyAST_MatchMapping(NULL, NULL, rest->v.Name.id, EXTRA) } + | '{' items=items_pattern ',' rest=double_star_pattern ','? '}' { _PyAST_MatchMapping( CHECK(asdl_expr_seq*, _PyPegen_get_pattern_keys(p, items)), CHECK(asdl_pattern_seq*, _PyPegen_get_patterns(p, items)), + rest->v.Name.id, + EXTRA) } + | '{' items=items_pattern ','? '}' { + _PyAST_MatchMapping( + CHECK(asdl_expr_seq*, _PyPegen_get_pattern_keys(p, items)), + CHECK(asdl_pattern_seq*, _PyPegen_get_patterns(p, items)), + NULL, EXTRA) } items_pattern[asdl_seq*]: - | items=','.key_value_pattern+ ','? { items } + | items=','.key_value_pattern+ { items } key_value_pattern[KeyPatternPair*]: | key=(literal_expr | attr) ':' pattern=pattern { _PyPegen_key_pattern_pair(p, key, pattern) } - | double_star_pattern -double_star_pattern[KeyPatternPair*]: - | '**' target=pattern_capture_target { - _PyPegen_key_pattern_pair(p, NULL, _PyAST_MatchAs(NULL, target->v.Name.id, EXTRA)) } +double_star_pattern[expr_ty]: + | '**' target=pattern_capture_target { target } class_pattern[pattern_ty]: | cls=name_or_attr '(' ')' { diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index e385730f7dc8d5..c75ac87973fe4b 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -581,6 +581,7 @@ struct _pattern { struct { asdl_expr_seq *keys; asdl_pattern_seq *patterns; + identifier rest; } MatchMapping; struct { @@ -818,8 +819,9 @@ pattern_ty _PyAST_MatchSequence(asdl_pattern_seq * patterns, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); pattern_ty _PyAST_MatchMapping(asdl_expr_seq * keys, asdl_pattern_seq * - patterns, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); + patterns, identifier rest, int lineno, int + col_offset, int end_lineno, int end_col_offset, + PyArena *arena); pattern_ty _PyAST_MatchClass(expr_ty cls, asdl_pattern_seq * patterns, asdl_identifier_seq * kwd_attrs, asdl_pattern_seq * kwd_patterns, int lineno, int col_offset, int diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index b7094d73885b75..e7c7b37d77b533 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -225,6 +225,7 @@ struct ast_state { PyObject *pattern_type; PyObject *patterns; PyObject *posonlyargs; + PyObject *rest; PyObject *returns; PyObject *right; PyObject *simple; diff --git a/Parser/Python.asdl b/Parser/Python.asdl index d0ce6b25dd1716..8a17e79268652f 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -130,11 +130,11 @@ module Python | MatchValue(expr value) | MatchSingleton(constant value) | MatchSequence(pattern* patterns) - | MatchMapping(expr* keys, pattern* patterns) + | MatchMapping(expr* keys, pattern* patterns, identifier? rest) | MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns) | MatchStar(identifier? target) - -- A NULL entry in the MatchMapping key list handles capturing extra mapping keys + -- The optional "rest" MatchMapping parameter handles capturing extra mapping keys | MatchAs(pattern? pattern, identifier target) | MatchOr(pattern* patterns) diff --git a/Parser/parser.c b/Parser/parser.c index 1556f02b3c034a..4244d02570326a 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -555,7 +555,7 @@ static pattern_ty star_pattern_rule(Parser *p); static pattern_ty mapping_pattern_rule(Parser *p); static asdl_seq* items_pattern_rule(Parser *p); static KeyPatternPair* key_value_pattern_rule(Parser *p); -static KeyPatternPair* double_star_pattern_rule(Parser *p); +static expr_ty double_star_pattern_rule(Parser *p); static pattern_ty class_pattern_rule(Parser *p); static asdl_pattern_seq* positional_patterns_rule(Parser *p); static asdl_seq* keyword_patterns_rule(Parser *p); @@ -7272,7 +7272,11 @@ star_pattern_rule(Parser *p) return _res; } -// mapping_pattern: '{' items_pattern? '}' +// mapping_pattern: +// | '{' '}' +// | '{' double_star_pattern ','? '}' +// | '{' items_pattern ',' double_star_pattern ','? '}' +// | '{' items_pattern ','? '}' static pattern_ty mapping_pattern_rule(Parser *p) { @@ -7292,24 +7296,156 @@ mapping_pattern_rule(Parser *p) UNUSED(_start_lineno); // Only used by EXTRA macro int _start_col_offset = p->tokens[_mark]->col_offset; UNUSED(_start_col_offset); // Only used by EXTRA macro - { // '{' items_pattern? '}' + { // '{' '}' if (p->error_indicator) { D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> mapping_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' items_pattern? '}'")); + D(fprintf(stderr, "%*c> mapping_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' '}'")); Token * _literal; Token * _literal_1; - void *items; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - (items = items_pattern_rule(p), 1) // items_pattern? + (_literal_1 = _PyPegen_expect_token(p, 26)) // token='}' + ) + { + D(fprintf(stderr, "%*c+ mapping_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' '}'")); + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + D(p->level--); + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_MatchMapping ( NULL , NULL , NULL , EXTRA ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + D(p->level--); + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s mapping_pattern[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' '}'")); + } + { // '{' double_star_pattern ','? '}' + if (p->error_indicator) { + D(p->level--); + return NULL; + } + D(fprintf(stderr, "%*c> mapping_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' double_star_pattern ','? '}'")); + Token * _literal; + Token * _literal_1; + void *_opt_var; + UNUSED(_opt_var); // Silence compiler warnings + expr_ty rest; + if ( + (_literal = _PyPegen_expect_token(p, 25)) // token='{' + && + (rest = double_star_pattern_rule(p)) // double_star_pattern + && + (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + && + (_literal_1 = _PyPegen_expect_token(p, 26)) // token='}' + ) + { + D(fprintf(stderr, "%*c+ mapping_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' double_star_pattern ','? '}'")); + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + D(p->level--); + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_MatchMapping ( NULL , NULL , rest -> v . Name . id , EXTRA ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + D(p->level--); + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s mapping_pattern[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' double_star_pattern ','? '}'")); + } + { // '{' items_pattern ',' double_star_pattern ','? '}' + if (p->error_indicator) { + D(p->level--); + return NULL; + } + D(fprintf(stderr, "%*c> mapping_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' items_pattern ',' double_star_pattern ','? '}'")); + Token * _literal; + Token * _literal_1; + Token * _literal_2; + void *_opt_var; + UNUSED(_opt_var); // Silence compiler warnings + asdl_seq* items; + expr_ty rest; + if ( + (_literal = _PyPegen_expect_token(p, 25)) // token='{' + && + (items = items_pattern_rule(p)) // items_pattern + && + (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' + && + (rest = double_star_pattern_rule(p)) // double_star_pattern + && + (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? + && + (_literal_2 = _PyPegen_expect_token(p, 26)) // token='}' + ) + { + D(fprintf(stderr, "%*c+ mapping_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' items_pattern ',' double_star_pattern ','? '}'")); + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + D(p->level--); + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_MatchMapping ( CHECK ( asdl_expr_seq * , _PyPegen_get_pattern_keys ( p , items ) ) , CHECK ( asdl_pattern_seq * , _PyPegen_get_patterns ( p , items ) ) , rest -> v . Name . id , EXTRA ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + D(p->level--); + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s mapping_pattern[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' items_pattern ',' double_star_pattern ','? '}'")); + } + { // '{' items_pattern ','? '}' + if (p->error_indicator) { + D(p->level--); + return NULL; + } + D(fprintf(stderr, "%*c> mapping_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' items_pattern ','? '}'")); + Token * _literal; + Token * _literal_1; + void *_opt_var; + UNUSED(_opt_var); // Silence compiler warnings + asdl_seq* items; + if ( + (_literal = _PyPegen_expect_token(p, 25)) // token='{' + && + (items = items_pattern_rule(p)) // items_pattern + && + (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? && (_literal_1 = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ mapping_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' items_pattern? '}'")); + D(fprintf(stderr, "%*c+ mapping_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' items_pattern ','? '}'")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { D(p->level--); @@ -7319,7 +7455,7 @@ mapping_pattern_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_MatchMapping ( CHECK ( asdl_expr_seq * , _PyPegen_get_pattern_keys ( p , items ) ) , CHECK ( asdl_pattern_seq * , _PyPegen_get_patterns ( p , items ) ) , EXTRA ); + _res = _PyAST_MatchMapping ( CHECK ( asdl_expr_seq * , _PyPegen_get_pattern_keys ( p , items ) ) , CHECK ( asdl_pattern_seq * , _PyPegen_get_patterns ( p , items ) ) , NULL , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); @@ -7329,7 +7465,7 @@ mapping_pattern_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s mapping_pattern[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' items_pattern? '}'")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{' items_pattern ','? '}'")); } _res = NULL; done: @@ -7337,7 +7473,7 @@ mapping_pattern_rule(Parser *p) return _res; } -// items_pattern: ','.key_value_pattern+ ','? +// items_pattern: ','.key_value_pattern+ static asdl_seq* items_pattern_rule(Parser *p) { @@ -7348,22 +7484,18 @@ items_pattern_rule(Parser *p) } asdl_seq* _res = NULL; int _mark = p->mark; - { // ','.key_value_pattern+ ','? + { // ','.key_value_pattern+ if (p->error_indicator) { D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> items_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.key_value_pattern+ ','?")); - void *_opt_var; - UNUSED(_opt_var); // Silence compiler warnings + D(fprintf(stderr, "%*c> items_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.key_value_pattern+")); asdl_seq * items; if ( (items = _gather_59_rule(p)) // ','.key_value_pattern+ - && - (_opt_var = _PyPegen_expect_token(p, 12), 1) // ','? ) { - D(fprintf(stderr, "%*c+ items_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.key_value_pattern+ ','?")); + D(fprintf(stderr, "%*c+ items_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.key_value_pattern+")); _res = items; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -7374,7 +7506,7 @@ items_pattern_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s items_pattern[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','.key_value_pattern+ ','?")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','.key_value_pattern+")); } _res = NULL; done: @@ -7382,7 +7514,7 @@ items_pattern_rule(Parser *p) return _res; } -// key_value_pattern: (literal_expr | attr) ':' pattern | double_star_pattern +// key_value_pattern: (literal_expr | attr) ':' pattern static KeyPatternPair* key_value_pattern_rule(Parser *p) { @@ -7423,25 +7555,6 @@ key_value_pattern_rule(Parser *p) D(fprintf(stderr, "%*c%s key_value_pattern[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(literal_expr | attr) ':' pattern")); } - { // double_star_pattern - if (p->error_indicator) { - D(p->level--); - return NULL; - } - D(fprintf(stderr, "%*c> key_value_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_star_pattern")); - KeyPatternPair* double_star_pattern_var; - if ( - (double_star_pattern_var = double_star_pattern_rule(p)) // double_star_pattern - ) - { - D(fprintf(stderr, "%*c+ key_value_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_star_pattern")); - _res = double_star_pattern_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s key_value_pattern[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_star_pattern")); - } _res = NULL; done: D(p->level--); @@ -7449,7 +7562,7 @@ key_value_pattern_rule(Parser *p) } // double_star_pattern: '**' pattern_capture_target -static KeyPatternPair* +static expr_ty double_star_pattern_rule(Parser *p) { D(p->level++); @@ -7457,17 +7570,8 @@ double_star_pattern_rule(Parser *p) D(p->level--); return NULL; } - KeyPatternPair* _res = NULL; + expr_ty _res = NULL; int _mark = p->mark; - if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { - p->error_indicator = 1; - D(p->level--); - return NULL; - } - int _start_lineno = p->tokens[_mark]->lineno; - UNUSED(_start_lineno); // Only used by EXTRA macro - int _start_col_offset = p->tokens[_mark]->col_offset; - UNUSED(_start_col_offset); // Only used by EXTRA macro { // '**' pattern_capture_target if (p->error_indicator) { D(p->level--); @@ -7483,16 +7587,7 @@ double_star_pattern_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ double_star_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' pattern_capture_target")); - Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); - if (_token == NULL) { - D(p->level--); - return NULL; - } - int _end_lineno = _token->end_lineno; - UNUSED(_end_lineno); // Only used by EXTRA macro - int _end_col_offset = _token->end_col_offset; - UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyPegen_key_pattern_pair ( p , NULL , _PyAST_MatchAs ( NULL , target -> v . Name . id , EXTRA ) ); + _res = target; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); diff --git a/Python/Python-ast.c b/Python/Python-ast.c index d21cd946fdaa4b..2853779bf14338 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -239,6 +239,7 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(state->pattern_type); Py_CLEAR(state->patterns); Py_CLEAR(state->posonlyargs); + Py_CLEAR(state->rest); Py_CLEAR(state->returns); Py_CLEAR(state->right); Py_CLEAR(state->simple); @@ -336,6 +337,7 @@ static int init_identifiers(struct ast_state *state) if ((state->pattern = PyUnicode_InternFromString("pattern")) == NULL) return 0; if ((state->patterns = PyUnicode_InternFromString("patterns")) == NULL) return 0; if ((state->posonlyargs = PyUnicode_InternFromString("posonlyargs")) == NULL) return 0; + if ((state->rest = PyUnicode_InternFromString("rest")) == NULL) return 0; if ((state->returns = PyUnicode_InternFromString("returns")) == NULL) return 0; if ((state->right = PyUnicode_InternFromString("right")) == NULL) return 0; if ((state->simple = PyUnicode_InternFromString("simple")) == NULL) return 0; @@ -723,6 +725,7 @@ static const char * const MatchSequence_fields[]={ static const char * const MatchMapping_fields[]={ "keys", "patterns", + "rest", }; static const char * const MatchClass_fields[]={ "cls", @@ -1776,7 +1779,7 @@ init_types(struct ast_state *state) " | MatchValue(expr value)\n" " | MatchSingleton(constant value)\n" " | MatchSequence(pattern* patterns)\n" - " | MatchMapping(expr* keys, pattern* patterns)\n" + " | MatchMapping(expr* keys, pattern* patterns, identifier? rest)\n" " | MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns)\n" " | MatchStar(identifier? target)\n" " | MatchAs(pattern? pattern, identifier target)\n" @@ -1805,9 +1808,11 @@ init_types(struct ast_state *state) if (!state->MatchSequence_type) return 0; state->MatchMapping_type = make_type(state, "MatchMapping", state->pattern_type, - MatchMapping_fields, 2, - "MatchMapping(expr* keys, pattern* patterns)"); + MatchMapping_fields, 3, + "MatchMapping(expr* keys, pattern* patterns, identifier? rest)"); if (!state->MatchMapping_type) return 0; + if (PyObject_SetAttr(state->MatchMapping_type, state->rest, Py_None) == -1) + return 0; state->MatchClass_type = make_type(state, "MatchClass", state->pattern_type, MatchClass_fields, 4, @@ -3467,9 +3472,9 @@ _PyAST_MatchSequence(asdl_pattern_seq * patterns, int lineno, int col_offset, } pattern_ty -_PyAST_MatchMapping(asdl_expr_seq * keys, asdl_pattern_seq * patterns, int - lineno, int col_offset, int end_lineno, int end_col_offset, - PyArena *arena) +_PyAST_MatchMapping(asdl_expr_seq * keys, asdl_pattern_seq * patterns, + identifier rest, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena) { pattern_ty p; p = (pattern_ty)_PyArena_Malloc(arena, sizeof(*p)); @@ -3478,6 +3483,7 @@ _PyAST_MatchMapping(asdl_expr_seq * keys, asdl_pattern_seq * patterns, int p->kind = MatchMapping_kind; p->v.MatchMapping.keys = keys; p->v.MatchMapping.patterns = patterns; + p->v.MatchMapping.rest = rest; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -5222,6 +5228,11 @@ ast2obj_pattern(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->patterns, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_identifier(state, o->v.MatchMapping.rest); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->rest, value) == -1) + goto failed; + Py_DECREF(value); break; case MatchClass_kind: tp = (PyTypeObject *)state->MatchClass_type; @@ -10426,6 +10437,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (isinstance) { asdl_expr_seq* keys; asdl_pattern_seq* patterns; + identifier rest; if (_PyObject_LookupAttr(obj, state->keys, &tmp) < 0) { return 1; @@ -10493,7 +10505,20 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, } Py_CLEAR(tmp); } - *out = _PyAST_MatchMapping(keys, patterns, lineno, col_offset, + if (_PyObject_LookupAttr(obj, state->rest, &tmp) < 0) { + return 1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + rest = NULL; + } + else { + int res; + res = obj2ast_identifier(state, tmp, &rest, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = _PyAST_MatchMapping(keys, patterns, rest, lineno, col_offset, end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; diff --git a/Python/ast.c b/Python/ast.c index 39d0d01d616bb0..a99fab55ccf738 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -442,14 +442,15 @@ validate_pattern(pattern_ty p) // return validate_patterns(p->v.MatchSequence.patterns); return 1; case MatchMapping_kind: + // TODO: check "rest" target name is valid if (asdl_seq_LEN(p->v.MatchMapping.keys) != asdl_seq_LEN(p->v.MatchMapping.patterns)) { PyErr_SetString(PyExc_ValueError, "MatchMapping doesn't have the same number of keys as patterns"); return 0; } - // null_ok=1 for key expressions to allow rest-of-mapping capture in patterns + // null_ok=0 for key expressions, as rest-of-mapping is captured in "rest" // TODO: replace with more restrictive expression validator, as per MatchValue above - if (!validate_exprs(p->v.MatchMapping.keys, Load, /*null_ok=*/ 1)) { + if (!validate_exprs(p->v.MatchMapping.keys, Load, /*null_ok=*/ 0)) { return 0; } // TODO: Validate all subpatterns @@ -470,9 +471,10 @@ validate_pattern(pattern_ty p) // validate_patterns(p->v.MatchClass.kwd_patterns); return 1; case MatchStar_kind: - // Nothing to check (except to potentially block "_" as an identifer) + // TODO: check target name is valid break; case MatchAs_kind: + // TODO: check target name is valid if (p->v.MatchAs.pattern) { return validate_pattern(p->v.MatchAs.pattern); } diff --git a/Python/compile.c b/Python/compile.c index 5654a6642160e6..7dc32a592b07d1 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -5934,12 +5934,13 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, pattern_context *pc) // just fail, don't crash out of the interpreter const char * e = "keys (%d) / patterns (%d) length mismatch in mapping pattern"; return compiler_error(c, e, size, npatterns); - } // A starred pattern will be a keyless value. It is guaranteed to be last: - int star = size ? !asdl_seq_GET(keys, size - 1) : 0; + } + // We have a double-star target if "rest" is set + PyObject *star_target = p->v.MatchMapping.rest; ADDOP(c, MATCH_MAPPING); ADDOP_JUMP(c, POP_JUMP_IF_FALSE, fail_pop_1); NEXT_BLOCK(c); - if (!size) { + if (!size && !star_target) { // If the pattern is just "{}", we're done! ADDOP(c, POP_TOP); ADDOP_LOAD_CONST(c, Py_True); @@ -5950,24 +5951,24 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, pattern_context *pc) compiler_use_next_block(c, end); return 1; } - if (size - star) { + if (size) { // If the pattern has any keys in it, perform a length check: ADDOP(c, GET_LEN); - ADDOP_LOAD_CONST_NEW(c, PyLong_FromSsize_t(size - star)); + ADDOP_LOAD_CONST_NEW(c, PyLong_FromSsize_t(size)); ADDOP_COMPARE(c, GtE); ADDOP_JUMP(c, POP_JUMP_IF_FALSE, fail_pop_1); NEXT_BLOCK(c); } - if (INT_MAX < size - star - 1) { + if (INT_MAX < size - 1) { return compiler_error(c, "too many sub-patterns in mapping pattern"); } // Collect all of the keys into a tuple for MATCH_KEYS and // COPY_DICT_WITHOUT_KEYS. They can either be dotted names or literals: - for (Py_ssize_t i = 0; i < size - star; i++) { + for (Py_ssize_t i = 0; i < size; i++) { expr_ty key = asdl_seq_GET(keys, i); if (key == NULL) { - const char *e = "can't use starred name here " - "(consider moving to end)"; + const char *e = "can't use NULL keys in MatchMapping " + "(set 'rest' parameter instead)"; c->u->u_col_offset = ((pattern_ty) asdl_seq_GET(patterns, i))->col_offset; return compiler_error(c, e); } @@ -5977,13 +5978,13 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, pattern_context *pc) } VISIT(c, expr, key); } - ADDOP_I(c, BUILD_TUPLE, size - star); + ADDOP_I(c, BUILD_TUPLE, size); ADDOP(c, MATCH_KEYS); ADDOP_JUMP(c, POP_JUMP_IF_FALSE, fail_pop_3); NEXT_BLOCK(c); // So far so good. There's now a tuple of values on the stack to match // sub-patterns against: - for (Py_ssize_t i = 0; i < size - star; i++) { + for (Py_ssize_t i = 0; i < size; i++) { pattern_ty pattern = asdl_seq_GET(patterns, i); if (WILDCARD_CHECK(pattern)) { continue; @@ -5997,11 +5998,10 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, pattern_context *pc) } // If we get this far, it's a match! We're done with that tuple of values. ADDOP(c, POP_TOP); - if (star) { - // If we had a starred name, bind a dict of remaining items to it: + if (star_target) { + // If we have a starred name, bind a dict of remaining items to it: ADDOP(c, COPY_DICT_WITHOUT_KEYS); - identifier id = asdl_seq_GET(patterns, size - 1)->v.MatchAs.target; - RETURN_IF_FALSE(pattern_helper_store_name(c, id, pc)); + RETURN_IF_FALSE(pattern_helper_store_name(c, star_target, pc)); } else { // Otherwise, we don't care about this tuple of keys anymore: diff --git a/Python/symtable.c b/Python/symtable.c index 1661e2cdbfafa5..cef84fb4398e23 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1689,11 +1689,8 @@ symtable_visit_pattern(struct symtable *st, pattern_ty p) VISIT_SEQ(st, pattern, p->v.MatchSequence.patterns); break; case MatchStar_kind: - { - PyObject *target_name = p->v.MatchStar.target; - if (target_name) { - symtable_add_def(st, target_name, DEF_LOCAL); - } + if (p->v.MatchStar.target) { + symtable_add_def(st, p->v.MatchStar.target, DEF_LOCAL); } break; case MatchMapping_kind: @@ -1701,6 +1698,9 @@ symtable_visit_pattern(struct symtable *st, pattern_ty p) VISIT_SEQ_WITH_NULL(st, expr, p->v.MatchMapping.keys); VISIT_SEQ(st, pattern, p->v.MatchMapping.patterns); } + if (p->v.MatchMapping.rest) { + symtable_add_def(st, p->v.MatchMapping.rest, DEF_LOCAL); + } break; case MatchClass_kind: VISIT(st, expr, p->v.MatchClass.cls); From d6e69f00a9e393fec32ac4eecff9ea3142393e54 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 25 Apr 2021 17:07:38 +1000 Subject: [PATCH 13/38] Use 'name' for identifier AST fields, not 'target' --- Include/internal/pycore_ast.h | 10 +++---- Lib/ast.py | 10 +++++-- Parser/Python.asdl | 4 +-- Python/Python-ast.c | 56 +++++++++++++++++------------------ Python/compile.c | 10 +++---- Python/symtable.c | 6 ++-- 6 files changed, 50 insertions(+), 46 deletions(-) diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index c75ac87973fe4b..e775a79708d456 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -592,12 +592,12 @@ struct _pattern { } MatchClass; struct { - identifier target; + identifier name; } MatchStar; struct { pattern_ty pattern; - identifier target; + identifier name; } MatchAs; struct { @@ -826,10 +826,10 @@ pattern_ty _PyAST_MatchClass(expr_ty cls, asdl_pattern_seq * patterns, asdl_identifier_seq * kwd_attrs, asdl_pattern_seq * kwd_patterns, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); -pattern_ty _PyAST_MatchStar(identifier target, int lineno, int col_offset, int +pattern_ty _PyAST_MatchStar(identifier name, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); -pattern_ty _PyAST_MatchAs(pattern_ty pattern, identifier target, int lineno, - int col_offset, int end_lineno, int end_col_offset, +pattern_ty _PyAST_MatchAs(pattern_ty pattern, identifier name, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena *arena); pattern_ty _PyAST_MatchOr(asdl_pattern_seq * patterns, int lineno, int col_offset, int end_lineno, int end_col_offset, diff --git a/Lib/ast.py b/Lib/ast.py index 703f68ace6068f..02d4d15d0e2e4a 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -1589,9 +1589,13 @@ def visit_match_case(self, node): def visit_MatchAs(self, node): with self.require_parens(_Precedence.TEST, node): - self.set_precedence(_Precedence.BOR, node.pattern) - self.traverse(node.pattern) - self.write(f" as {node.name}") + pattern = node.pattern + if pattern is not None: + self.set_precedence(_Precedence.BOR, node.pattern) + self.traverse(node.pattern) + self.write(f" as {node.name}") + else: + self.write(node.name) def visit_MatchOr(self, node): with self.require_parens(_Precedence.BOR, node): diff --git a/Parser/Python.asdl b/Parser/Python.asdl index 8a17e79268652f..dec16bda5fce11 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -133,10 +133,10 @@ module Python | MatchMapping(expr* keys, pattern* patterns, identifier? rest) | MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns) - | MatchStar(identifier? target) + | MatchStar(identifier? name) -- The optional "rest" MatchMapping parameter handles capturing extra mapping keys - | MatchAs(pattern? pattern, identifier target) + | MatchAs(pattern? pattern, identifier name) | MatchOr(pattern* patterns) attributes (int lineno, int col_offset, int end_lineno, int end_col_offset) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 2853779bf14338..d888538682d80b 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -734,11 +734,11 @@ static const char * const MatchClass_fields[]={ "kwd_patterns", }; static const char * const MatchStar_fields[]={ - "target", + "name", }; static const char * const MatchAs_fields[]={ "pattern", - "target", + "name", }; static const char * const MatchOr_fields[]={ "patterns", @@ -1781,8 +1781,8 @@ init_types(struct ast_state *state) " | MatchSequence(pattern* patterns)\n" " | MatchMapping(expr* keys, pattern* patterns, identifier? rest)\n" " | MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns)\n" - " | MatchStar(identifier? target)\n" - " | MatchAs(pattern? pattern, identifier target)\n" + " | MatchStar(identifier? name)\n" + " | MatchAs(pattern? pattern, identifier name)\n" " | MatchOr(pattern* patterns)"); if (!state->pattern_type) return 0; if (!add_attributes(state, state->pattern_type, pattern_attributes, 4)) @@ -1820,13 +1820,13 @@ init_types(struct ast_state *state) if (!state->MatchClass_type) return 0; state->MatchStar_type = make_type(state, "MatchStar", state->pattern_type, MatchStar_fields, 1, - "MatchStar(identifier? target)"); + "MatchStar(identifier? name)"); if (!state->MatchStar_type) return 0; - if (PyObject_SetAttr(state->MatchStar_type, state->target, Py_None) == -1) + if (PyObject_SetAttr(state->MatchStar_type, state->name, Py_None) == -1) return 0; state->MatchAs_type = make_type(state, "MatchAs", state->pattern_type, MatchAs_fields, 2, - "MatchAs(pattern? pattern, identifier target)"); + "MatchAs(pattern? pattern, identifier name)"); if (!state->MatchAs_type) return 0; if (PyObject_SetAttr(state->MatchAs_type, state->pattern, Py_None) == -1) return 0; @@ -3519,7 +3519,7 @@ _PyAST_MatchClass(expr_ty cls, asdl_pattern_seq * patterns, asdl_identifier_seq } pattern_ty -_PyAST_MatchStar(identifier target, int lineno, int col_offset, int end_lineno, +_PyAST_MatchStar(identifier name, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) { pattern_ty p; @@ -3527,7 +3527,7 @@ _PyAST_MatchStar(identifier target, int lineno, int col_offset, int end_lineno, if (!p) return NULL; p->kind = MatchStar_kind; - p->v.MatchStar.target = target; + p->v.MatchStar.name = name; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -3536,13 +3536,13 @@ _PyAST_MatchStar(identifier target, int lineno, int col_offset, int end_lineno, } pattern_ty -_PyAST_MatchAs(pattern_ty pattern, identifier target, int lineno, int - col_offset, int end_lineno, int end_col_offset, PyArena *arena) +_PyAST_MatchAs(pattern_ty pattern, identifier name, int lineno, int col_offset, + int end_lineno, int end_col_offset, PyArena *arena) { pattern_ty p; - if (!target) { + if (!name) { PyErr_SetString(PyExc_ValueError, - "field 'target' is required for MatchAs"); + "field 'name' is required for MatchAs"); return NULL; } p = (pattern_ty)_PyArena_Malloc(arena, sizeof(*p)); @@ -3550,7 +3550,7 @@ _PyAST_MatchAs(pattern_ty pattern, identifier target, int lineno, int return NULL; p->kind = MatchAs_kind; p->v.MatchAs.pattern = pattern; - p->v.MatchAs.target = target; + p->v.MatchAs.name = name; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -5266,9 +5266,9 @@ ast2obj_pattern(struct ast_state *state, void* _o) tp = (PyTypeObject *)state->MatchStar_type; result = PyType_GenericNew(tp, NULL, NULL); if (!result) goto failed; - value = ast2obj_identifier(state, o->v.MatchStar.target); + value = ast2obj_identifier(state, o->v.MatchStar.name); if (!value) goto failed; - if (PyObject_SetAttr(result, state->target, value) == -1) + if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); break; @@ -5281,9 +5281,9 @@ ast2obj_pattern(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->pattern, value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_identifier(state, o->v.MatchAs.target); + value = ast2obj_identifier(state, o->v.MatchAs.name); if (!value) goto failed; - if (PyObject_SetAttr(result, state->target, value) == -1) + if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); break; @@ -10658,22 +10658,22 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, return 1; } if (isinstance) { - identifier target; + identifier name; - if (_PyObject_LookupAttr(obj, state->target, &tmp) < 0) { + if (_PyObject_LookupAttr(obj, state->name, &tmp) < 0) { return 1; } if (tmp == NULL || tmp == Py_None) { Py_CLEAR(tmp); - target = NULL; + name = NULL; } else { int res; - res = obj2ast_identifier(state, tmp, &target, arena); + res = obj2ast_identifier(state, tmp, &name, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = _PyAST_MatchStar(target, lineno, col_offset, end_lineno, + *out = _PyAST_MatchStar(name, lineno, col_offset, end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; @@ -10685,7 +10685,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, } if (isinstance) { pattern_ty pattern; - identifier target; + identifier name; if (_PyObject_LookupAttr(obj, state->pattern, &tmp) < 0) { return 1; @@ -10700,20 +10700,20 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (res != 0) goto failed; Py_CLEAR(tmp); } - if (_PyObject_LookupAttr(obj, state->target, &tmp) < 0) { + if (_PyObject_LookupAttr(obj, state->name, &tmp) < 0) { return 1; } if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from MatchAs"); + PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from MatchAs"); return 1; } else { int res; - res = obj2ast_identifier(state, tmp, &target, arena); + res = obj2ast_identifier(state, tmp, &name, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = _PyAST_MatchAs(pattern, target, lineno, col_offset, end_lineno, + *out = _PyAST_MatchAs(pattern, name, lineno, col_offset, end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; diff --git a/Python/compile.c b/Python/compile.c index abff41331dae92..cf4abf64ac1adc 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -5600,7 +5600,7 @@ compiler_slice(struct compiler *c, expr_ty s) ((N)->kind == MatchAlways_kind) #define WILDCARD_STAR_CHECK(N) \ - ((N)->kind == MatchStar_kind && !(N)->v.MatchStar.target) + ((N)->kind == MatchStar_kind && !(N)->v.MatchStar.name) // Limit permitted subexpressions, even if the parser & AST validator let them through #define MATCH_VALUE_EXPR(N) \ @@ -5798,9 +5798,9 @@ compiler_pattern_as(struct compiler *c, pattern_ty p, pattern_context *pc) if (!pc->allow_irrefutable) { // Whoops, can't have a name capture here! const char *e = "name capture %R makes remaining patterns unreachable"; - return compiler_error(c, e, p->v.MatchAs.target); + return compiler_error(c, e, p->v.MatchAs.name); } - return compiler_pattern_capture(c, p->v.MatchAs.target, pc); + return compiler_pattern_capture(c, p->v.MatchAs.name, pc); } basicblock *end, *fail_pop_1; RETURN_IF_FALSE(end = compiler_new_block(c)); @@ -5810,7 +5810,7 @@ compiler_pattern_as(struct compiler *c, pattern_ty p, pattern_context *pc) RETURN_IF_FALSE(compiler_pattern(c, p->v.MatchAs.pattern, pc)); ADDOP_JUMP(c, POP_JUMP_IF_FALSE, fail_pop_1); NEXT_BLOCK(c); - RETURN_IF_FALSE(pattern_helper_store_name(c, p->v.MatchAs.target, pc)); + RETURN_IF_FALSE(pattern_helper_store_name(c, p->v.MatchAs.name, pc)); ADDOP_LOAD_CONST(c, Py_True); ADDOP_JUMP(c, JUMP_FORWARD, end); compiler_use_next_block(c, fail_pop_1); @@ -5830,7 +5830,7 @@ compiler_pattern_star(struct compiler *c, pattern_ty p, pattern_context *pc) const char *e = "star captures are only allowed as part of sequence patterns"; return compiler_error(c, e); } - return compiler_pattern_capture(c, p->v.MatchStar.target, pc); + return compiler_pattern_capture(c, p->v.MatchStar.name, pc); } static int diff --git a/Python/symtable.c b/Python/symtable.c index 1c03bc16d84f25..905f0e22353ef8 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1721,8 +1721,8 @@ symtable_visit_pattern(struct symtable *st, pattern_ty p) VISIT_SEQ(st, pattern, p->v.MatchSequence.patterns); break; case MatchStar_kind: - if (p->v.MatchStar.target) { - symtable_add_def(st, p->v.MatchStar.target, DEF_LOCAL); + if (p->v.MatchStar.name) { + symtable_add_def(st, p->v.MatchStar.name, DEF_LOCAL); } break; case MatchMapping_kind: @@ -1747,7 +1747,7 @@ symtable_visit_pattern(struct symtable *st, pattern_ty p) if (p->v.MatchAs.pattern) { VISIT(st, pattern, p->v.MatchAs.pattern); } - symtable_add_def(st, p->v.MatchAs.target, DEF_LOCAL); + symtable_add_def(st, p->v.MatchAs.name, DEF_LOCAL); break; case MatchOr_kind: VISIT_SEQ(st, pattern, p->v.MatchOr.patterns); From 5a0b4f24b4789227ee6513dd4bdddd8afced4655 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 25 Apr 2021 18:30:44 +1000 Subject: [PATCH 14/38] Implement unparsing --- Lib/ast.py | 71 ++++++++++++++++++++++++++++++++++++++++---- Python/ast_unparse.c | 13 +++++--- 2 files changed, 75 insertions(+), 9 deletions(-) diff --git a/Lib/ast.py b/Lib/ast.py index 02d4d15d0e2e4a..b4fcf6f0bd677c 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -798,6 +798,9 @@ def traverse(self, node): else: super().visit(node) + # Note: as visit() resets the output text, do NOT rely on + # NodeVisitor.generic_visit to handle any nodes (as it calls back in to + # the subclass visit() method, which resets self._source to an empty list) def visit(self, node): """Outputs a source code string that, if converted back to an ast (using ast.parse) will generate an AST equivalent to *node*""" @@ -1587,15 +1590,73 @@ def visit_match_case(self, node): with self.block(): self.traverse(node.body) + def visit_MatchAlways(self, node): + self.write("_") + + def visit_MatchValue(self, node): + self.traverse(node.value) + + def visit_MatchSingleton(self, node): + self._write_constant(node.value) + + def visit_MatchSequence(self, node): + with self.delimit("[", "]"): + self.interleave(lambda: self.write(", "), self.traverse, node.patterns) + + def visit_MatchStar(self, node): + name = node.name + if name is None: + name = "_" + self.write(f"*{name}") + + def visit_MatchMapping(self, node): + def write_key_pattern_pair(pair): + k, p = pair + self.traverse(k) + self.write(": ") + self.traverse(p) + + with self.delimit("{", "}"): + keys = node.keys + self.interleave( + lambda: self.write(", "), write_key_pattern_pair, zip(keys, node.patterns) + ) + rest = node.rest + if rest is not None: + if keys: + self.write(", ") + self.write(f"**{rest}") + + def visit_MatchClass(self, node): + self.set_precedence(_Precedence.ATOM, node.cls) + self.traverse(node.cls) + with self.delimit("(", ")"): + patterns = node.patterns + self.interleave( + lambda: self.write(", "), self.traverse, patterns + ) + attrs = node.kwd_attrs + if attrs: + def write_attr_pattern(pair): + attr, pattern = pair + self.write(f"{attr}=") + self.traverse(pattern) + + if patterns: + self.write(", ") + self.interleave( + lambda: self.write(", "), write_attr_pattern, zip(attrs, node.kwd_patterns) + ) + def visit_MatchAs(self, node): - with self.require_parens(_Precedence.TEST, node): - pattern = node.pattern - if pattern is not None: + pattern = node.pattern + if pattern is not None: + with self.require_parens(_Precedence.TEST, node): self.set_precedence(_Precedence.BOR, node.pattern) self.traverse(node.pattern) self.write(f" as {node.name}") - else: - self.write(node.name) + else: + self.write(node.name) def visit_MatchOr(self, node): with self.require_parens(_Precedence.BOR, node): diff --git a/Python/ast_unparse.c b/Python/ast_unparse.c index 5276b2fcd6698b..126e9047d58d64 100644 --- a/Python/ast_unparse.c +++ b/Python/ast_unparse.c @@ -3,6 +3,11 @@ #include // DBL_MAX_10_EXP #include +/* This limited unparser is used to convert annotations back to strings + * during compilation rather than being a full AST unparser. + * See ast.unparse for a full unparser (written in Python) + */ + static PyObject *_str_open_br; static PyObject *_str_dbl_open_br; static PyObject *_str_close_br; @@ -912,11 +917,11 @@ append_ast_expr(_PyUnicodeWriter *writer, expr_ty e, int level) return append_ast_tuple(writer, e, level); case NamedExpr_kind: return append_named_expr(writer, e, level); - default: - PyErr_SetString(PyExc_SystemError, - "unknown expression kind"); - return -1; + // No default so compiler emits a warning for unhandled cases } + PyErr_SetString(PyExc_SystemError, + "unknown expression kind"); + return -1; } static int From e07781ed7b284160808c9b6421e002a647e239c1 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 25 Apr 2021 18:40:34 +1000 Subject: [PATCH 15/38] Add news entry --- .../2021-04-25-08-35-11.bpo-43892.hr5Ke2.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-04-25-08-35-11.bpo-43892.hr5Ke2.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-04-25-08-35-11.bpo-43892.hr5Ke2.rst b/Misc/NEWS.d/next/Core and Builtins/2021-04-25-08-35-11.bpo-43892.hr5Ke2.rst new file mode 100644 index 00000000000000..65997548df2138 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-04-25-08-35-11.bpo-43892.hr5Ke2.rst @@ -0,0 +1,5 @@ +Match patterns now use new dedicated AST nodes (``MatchAlways``, ``MatchValue``, +``MatchSingleton``, ``MatchSequence``, ``MatchStar``, ``MatchMapping``, +``MatchClass``) rather than reusing expression AST nodes. ``MatchAs`` and +``MatchOr`` are now defined as pattern nodes rather than as expression nodes. +Patch by Nick Coghlan. From b17e7f4ade04fe8583bfd0e0c3ecaecb9e610c74 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 26 Apr 2021 22:31:49 +1000 Subject: [PATCH 16/38] Update AST documentation --- Doc/library/ast.rst | 309 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 282 insertions(+), 27 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index ff3fd99f5d4f44..0615ef6460f430 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -1245,6 +1245,10 @@ Control flow type_ignores=[]) +Pattern matching +^^^^^^^^^^^^^^^^ + + .. class:: Match(subject, cases) A ``match`` statement. ``subject`` holds the subject of the match (the object @@ -1254,14 +1258,14 @@ Control flow .. class:: match_case(pattern, guard, body) - A single case pattern in a ``match`` statement. ``pattern`` contains the - match pattern that will be used to match the subject against. Notice that - the meaning of the :class:`AST` nodes in this attribute have a different - meaning than in other places, as they represent patterns to match against. - The ``guard`` attribute contains an expression that will be evaluated if - the pattern matches the subject. If the pattern matches and the ``guard`` condition - is truthy, the body of the case shall be executed. ``body`` contains a list - of nodes to execute if the guard is truthy. + A single case pattern in a ``match`` statement. ``pattern`` contains the + match pattern that will be used to match the subject against. Notice that + the meaning of the :class:`AST` nodes in this attribute have a different + meaning than in other places, as they represent patterns to match against. + The ``guard`` attribute contains an expression that will be evaluated if + the pattern matches the subject. If the pattern matches and the ``guard`` condition + is truthy, the body of the case shall be executed. ``body`` contains a list + of nodes to execute if the guard is truthy. .. doctest:: @@ -1301,12 +1305,266 @@ Control flow value=Constant(value=Ellipsis))])])], type_ignores=[]) +.. class:: MatchAlways() + + A match wildcard pattern. This pattern always succeeds with no runtime + effect. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case _: + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchAlways(), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + +.. class:: MatchValue(value) + + A match literal or value pattern that compares by equality. ``value`` is + an expression node. Permitted value nodes are restricted as described in + the match statement documentation. This pattern succeeds if the match + subject is equal to the evaluated value. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case "Relevant": + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchValue( + value=Constant(value='Relevant')), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + +.. class:: MatchSingleton(value) + + A match literal pattern that compares by identity. ``value`` is the + singleton to be compared against: ``None``, ``True``, or ``False``. This + pattern succeeds if the match subject is the given constant. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case None: + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchSingleton(value=None), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + +.. class:: MatchSequence(patterns) + + A match sequence pattern. ``patterns`` contains the patterns to be matched + against the subject elements if the subject is a sequence. Matches a variable + length sequence if one of the subpatterns is a ``MatchStar`` node, otherwise + matches a fixed length sequence. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case [1, 2]: + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchSequence( + patterns=[ + MatchValue( + value=Constant(value=1)), + MatchValue( + value=Constant(value=2))]), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + +.. class:: MatchStar(name) + + Matches the rest of the sequence in a variable length match sequence pattern. + If ``name`` is not ``None``, a tuple containing the remaining sequence + elements is bound to that name if the overall sequence pattern is successful. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case [1, 2, *rest]: + ... ... + ... case [*_]: + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchSequence( + patterns=[ + MatchValue( + value=Constant(value=1)), + MatchValue( + value=Constant(value=2)), + MatchStar(name='rest')]), + body=[ + Expr( + value=Constant(value=Ellipsis))]), + match_case( + pattern=MatchSequence( + patterns=[ + MatchStar()]), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + +.. class:: MatchMapping(keys, patterns, rest) + + A match mapping pattern. ``keys`` is a sequence of expression nodes. + ``patterns`` is a corresponding sequence of pattern nodes. ``rest`` is an + optional name that can be specified to capture the remaining mapping elements. + Permitted key expressions are restricted as described in the match statement + documentation. + + This pattern succeeds if the subject is a mapping, all evaluated key + expressions are present in the mapping, and the value corresponding to each + key matches the corresponding subpattern. If ``rest`` is not ``None``, a dict + containing the remaining mapping elements is bound to that name if the overall + mapping pattern is successful. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case {1:_, 2:_}: + ... ... + ... case {**rest}: + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchMapping( + keys=[ + Constant(value=1), + Constant(value=2)], + patterns=[ + MatchAlways(), + MatchAlways()]), + body=[ + Expr( + value=Constant(value=Ellipsis))]), + match_case( + pattern=MatchMapping(keys=[], patterns=[], rest='rest'), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + +.. class:: MatchClass(cls, patterns, kwd_attrs, kwd_patterns) + + A match class pattern. ``cls`` is the nominal class to be matched. + ``patterns`` is a sequence of pattern nodes to be matched against the class + defined sequence of pattern matching attributes. ``kwd_attrs`` is a sequence + of additional attributes to be matched (specified as keyword arguments in the + class pattern), ``kwd_patterns`` are the corresponding patterns (specified as + keyword values in the class pattern). + + This pattern succeeds if the subject is an instance of the nominated class, + all positional patterns match the corresponding class-defined attributes, and + any specified keyword attributes match their corresponding pattern. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case Point2D(0, 0): + ... ... + ... case Point3D(x=0, y=0, z=0): + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchClass( + cls=Name(id='Point2D', ctx=Load()), + patterns=[ + MatchValue( + value=Constant(value=0)), + MatchValue( + value=Constant(value=0))], + kwd_attrs=[], + kwd_patterns=[]), + body=[ + Expr( + value=Constant(value=Ellipsis))]), + match_case( + pattern=MatchClass( + cls=Name(id='Point3D', ctx=Load()), + patterns=[], + kwd_attrs=[ + 'x', + 'y', + 'z'], + kwd_patterns=[ + MatchValue( + value=Constant(value=0)), + MatchValue( + value=Constant(value=0)), + MatchValue( + value=Constant(value=0))]), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + .. class:: MatchAs(pattern, name) - A match "as-pattern". The as-pattern matches whatever pattern is on its - left-hand side, but also binds the value to a name. ``pattern`` contains - the match pattern that will be used to match the subject agsinst. The ``name`` - attribute contains the name that will be binded if the pattern is successful. + A match "as-pattern" or capture pattern. ``pattern`` contains the match + pattern that will be used to match the subject against. If the pattern is + ``None``, the node represents a capture pattern (i.e a bare name) and will + always succeed. The ``name`` attribute contains the name that will be bound + if the pattern is successful. .. doctest:: @@ -1322,24 +1580,22 @@ Control flow cases=[ match_case( pattern=MatchAs( - pattern=List( - elts=[ - Name(id='x', ctx=Store())], - ctx=Load()), + pattern=MatchSequence( + patterns=[ + MatchAs(name='x')]), name='y'), body=[ Expr( value=Constant(value=Ellipsis))])])], type_ignores=[]) - .. class:: MatchOr(patterns) - A match "or-pattern". An or-pattern matches each of its subpatterns in turn - to the subject, until one succeeds. The or-pattern is then deemed to - succeed. If none of the subpatterns succeed the or-pattern fails. The - ``patterns`` attribute contains a list of match patterns nodes that will be - matched against the subject. + A match "or-pattern". An or-pattern matches each of its subpatterns in turn + to the subject, until one succeeds. The or-pattern is then deemed to + succeed. If none of the subpatterns succeed the or-pattern fails. The + ``patterns`` attribute contains a list of match patterns nodes that will be + matched against the subject. .. doctest:: @@ -1356,11 +1612,10 @@ Control flow match_case( pattern=MatchOr( patterns=[ - List( - elts=[ - Name(id='x', ctx=Store())], - ctx=Load()), - Name(id='y', ctx=Store())]), + MatchSequence( + patterns=[ + MatchAs(name='x')]), + MatchAs(name='y')]), body=[ Expr( value=Constant(value=Ellipsis))])])], From 466fccf740d7d9c7bd3195b0e18455bd3e862522 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 26 Apr 2021 22:38:28 +1000 Subject: [PATCH 17/38] Update recursion depth for nested patterns in AST optimiser --- Python/ast_opt.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index ddba79d92753ba..d16b552addf303 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -798,6 +798,11 @@ astfold_pattern(pattern_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) // Currently, this is really only used to form complex/negative numeric // constants in MatchValue and MatchMapping nodes // We still recurse into all subexpressions and subpatterns anyway + if (++state->recursion_depth > state->recursion_limit) { + PyErr_SetString(PyExc_RecursionError, + "maximum recursion depth exceeded during compilation"); + return 0; + } switch (node_->kind) { case MatchAlways_kind: break; @@ -831,6 +836,7 @@ astfold_pattern(pattern_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) // No default case, so the compiler will emit a warning if new pattern // kinds are added without being handled here } + state->recursion_depth--; return 1; } From 6a9fbc80e5ea43c1874cbdc07ecee7ad6b63b46f Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Mon, 26 Apr 2021 23:32:00 +1000 Subject: [PATCH 18/38] Fix remaining AST doctest --- Doc/library/ast.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 0615ef6460f430..e184e2c670687c 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -1282,10 +1282,9 @@ Pattern matching subject=Name(id='x', ctx=Load()), cases=[ match_case( - pattern=List( - elts=[ - Name(id='x', ctx=Store())], - ctx=Load()), + pattern=MatchSequence( + patterns=[ + MatchAs(name='x')]), guard=Compare( left=Name(id='x', ctx=Load()), ops=[ @@ -1296,10 +1295,11 @@ Pattern matching Expr( value=Constant(value=Ellipsis))]), match_case( - pattern=Call( - func=Name(id='tuple', ctx=Load()), - args=[], - keywords=[]), + pattern=MatchClass( + cls=Name(id='tuple', ctx=Load()), + patterns=[], + kwd_attrs=[], + kwd_patterns=[]), body=[ Expr( value=Constant(value=Ellipsis))])])], From e8a181ab94309a9ab6b1fa7feca99c1d5439b109 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Tue, 27 Apr 2021 00:25:19 +1000 Subject: [PATCH 19/38] Validate the AST emitted by the parser --- Python/bltinmodule.c | 6 +----- Python/pythonrun.c | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 3b0e59a6d18c36..66a74cbdef6104 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -831,11 +831,7 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, if (arena == NULL) goto error; mod = PyAST_obj2mod(source, arena, compile_mode); - if (mod == NULL) { - _PyArena_Free(arena); - goto error; - } - if (!_PyAST_Validate(mod)) { + if (mod == NULL || !_PyAST_Validate(mod)) { _PyArena_Free(arena); goto error; } diff --git a/Python/pythonrun.c b/Python/pythonrun.c index f00e3eb0de803f..75af7ceb58949f 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1366,7 +1366,7 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, return NULL; mod = _PyParser_ASTFromString(str, filename, start, flags, arena); - if (mod == NULL) { + if (mod == NULL || !_PyAST_Validate(mod)) { _PyArena_Free(arena); return NULL; } From aa429c25b82855975e89ff0059c3a1b226fe862b Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Tue, 27 Apr 2021 00:53:29 +1000 Subject: [PATCH 20/38] AST validator did not handle FunctionType nodes --- Python/ast.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index dd1ce7b8ed4484..f5eefce6f924eb 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -824,7 +824,7 @@ validate_exprs(struct validator *state, asdl_expr_seq *exprs, expr_context_ty ct int _PyAST_Validate(mod_ty mod) { - int res = 0; + int res = -1; struct validator state; PyThreadState *tstate; int recursion_limit = Py_GetRecursionLimit(); @@ -852,10 +852,16 @@ _PyAST_Validate(mod_ty mod) case Expression_kind: res = validate_expr(&state, mod->v.Expression.body, Load); break; - default: - PyErr_SetString(PyExc_SystemError, "impossible module node"); - res = 0; + case FunctionType_kind: + res = validate_exprs(&state, mod->v.FunctionType.argtypes, Load, /*null_ok=*/0) && + validate_expr(&state, mod->v.FunctionType.returns, Load); break; + // No default case so compiler emits warning for unhandled cases + } + + if (res < 0) { + PyErr_SetString(PyExc_SystemError, "impossible module node"); + return 0; } /* Check that the recursion depth counting balanced correctly */ From e4dfa5a74cf253b0bdb45189a3c534f30c4fd002 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Tue, 27 Apr 2021 21:33:32 +1000 Subject: [PATCH 21/38] Apply suggestions from code review Co-authored-by: Brandt Bucher --- Doc/library/ast.rst | 4 ++-- Lib/ast.py | 4 ++-- Python/symtable.c | 14 ++++---------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index e184e2c670687c..781d77a687ad16 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -1415,7 +1415,7 @@ Pattern matching .. class:: MatchStar(name) Matches the rest of the sequence in a variable length match sequence pattern. - If ``name`` is not ``None``, a tuple containing the remaining sequence + If ``name`` is not ``None``, a list containing the remaining sequence elements is bound to that name if the overall sequence pattern is successful. .. doctest:: @@ -1470,7 +1470,7 @@ Pattern matching >>> print(ast.dump(ast.parse(""" ... match x: - ... case {1:_, 2:_}: + ... case {1: _, 2: _}: ... ... ... case {**rest}: ... ... diff --git a/Lib/ast.py b/Lib/ast.py index b4fcf6f0bd677c..ccb341ed06d0c1 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -1619,7 +1619,7 @@ def write_key_pattern_pair(pair): with self.delimit("{", "}"): keys = node.keys self.interleave( - lambda: self.write(", "), write_key_pattern_pair, zip(keys, node.patterns) + lambda: self.write(", "), write_key_pattern_pair, zip(keys, node.patterns, strict=True) ) rest = node.rest if rest is not None: @@ -1645,7 +1645,7 @@ def write_attr_pattern(pair): if patterns: self.write(", ") self.interleave( - lambda: self.write(", "), write_attr_pattern, zip(attrs, node.kwd_patterns) + lambda: self.write(", "), write_attr_pattern, zip(attrs, node.kwd_patterns, strict=True) ) def visit_MatchAs(self, node): diff --git a/Python/symtable.c b/Python/symtable.c index 905f0e22353ef8..355bb8403f3902 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1726,22 +1726,16 @@ symtable_visit_pattern(struct symtable *st, pattern_ty p) } break; case MatchMapping_kind: - if (p->v.MatchMapping.keys) { - VISIT_SEQ_WITH_NULL(st, expr, p->v.MatchMapping.keys); - VISIT_SEQ(st, pattern, p->v.MatchMapping.patterns); - } + VISIT_SEQ(st, expr, p->v.MatchMapping.keys); + VISIT_SEQ(st, pattern, p->v.MatchMapping.patterns); if (p->v.MatchMapping.rest) { symtable_add_def(st, p->v.MatchMapping.rest, DEF_LOCAL); } break; case MatchClass_kind: VISIT(st, expr, p->v.MatchClass.cls); - if (p->v.MatchClass.patterns) { - VISIT_SEQ(st, pattern, p->v.MatchClass.patterns); - } - if (p->v.MatchClass.kwd_patterns) { - VISIT_SEQ(st, pattern, p->v.MatchClass.kwd_patterns); - } + VISIT_SEQ(st, pattern, p->v.MatchClass.patterns); + VISIT_SEQ(st, pattern, p->v.MatchClass.kwd_patterns); break; case MatchAs_kind: if (p->v.MatchAs.pattern) { From 2c289e7076e91ec96e1be7e975efbc2b2ff61884 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Tue, 27 Apr 2021 21:49:45 +1000 Subject: [PATCH 22/38] Replace MatchAlways() with MatchAs(NULL, NULL) --- Doc/example.ini | 13 +++++ Doc/library/ast.rst | 81 +++++++++++++--------------- Doc/myfile.bz2 | Bin 0 -> 331 bytes Grammar/python.gram | 2 +- Include/internal/pycore_ast.h | 10 ++-- Include/internal/pycore_ast_state.h | 1 - Lib/ast.py | 12 ++--- Parser/Python.asdl | 26 ++++----- Parser/parser.c | 2 +- Python/Python-ast.c | 61 +++------------------ Python/ast.c | 13 +++-- Python/ast_opt.c | 2 - Python/compile.c | 35 ++++++------ Python/symtable.c | 7 ++- 14 files changed, 111 insertions(+), 154 deletions(-) create mode 100644 Doc/example.ini create mode 100644 Doc/myfile.bz2 diff --git a/Doc/example.ini b/Doc/example.ini new file mode 100644 index 00000000000000..69e147f5825889 --- /dev/null +++ b/Doc/example.ini @@ -0,0 +1,13 @@ +[DEFAULT] +serveraliveinterval = 45 +compression = yes +compressionlevel = 9 +forwardx11 = yes + +[bitbucket.org] +user = hg + +[topsecret.server.com] +port = 50022 +forwardx11 = no + diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 781d77a687ad16..ba34e82e83453d 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -1255,17 +1255,18 @@ Pattern matching that is being matched against the cases) and ``cases`` contains an iterable of :class:`match_case` nodes with the different cases. - .. class:: match_case(pattern, guard, body) A single case pattern in a ``match`` statement. ``pattern`` contains the - match pattern that will be used to match the subject against. Notice that - the meaning of the :class:`AST` nodes in this attribute have a different - meaning than in other places, as they represent patterns to match against. + match pattern that the subject will be matched against. Note that the + :class:`AST` nodes produced for patterns differ from those produced for + expressions, even when they share the same syntax. + The ``guard`` attribute contains an expression that will be evaluated if - the pattern matches the subject. If the pattern matches and the ``guard`` condition - is truthy, the body of the case shall be executed. ``body`` contains a list - of nodes to execute if the guard is truthy. + the pattern matches the subject. + + ``body`` contains a list of nodes to execute if the pattern matches and + the result of evaluating the guard expression is truthy. .. doctest:: @@ -1305,30 +1306,6 @@ Pattern matching value=Constant(value=Ellipsis))])])], type_ignores=[]) -.. class:: MatchAlways() - - A match wildcard pattern. This pattern always succeeds with no runtime - effect. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... match x: - ... case _: - ... ... - ... """), indent=4)) - Module( - body=[ - Match( - subject=Name(id='x', ctx=Load()), - cases=[ - match_case( - pattern=MatchAlways(), - body=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - .. class:: MatchValue(value) A match literal or value pattern that compares by equality. ``value`` is @@ -1486,8 +1463,8 @@ Pattern matching Constant(value=1), Constant(value=2)], patterns=[ - MatchAlways(), - MatchAlways()]), + MatchAs(), + MatchAs()]), body=[ Expr( value=Constant(value=Ellipsis))]), @@ -1500,17 +1477,21 @@ Pattern matching .. class:: MatchClass(cls, patterns, kwd_attrs, kwd_patterns) - A match class pattern. ``cls`` is the nominal class to be matched. - ``patterns`` is a sequence of pattern nodes to be matched against the class - defined sequence of pattern matching attributes. ``kwd_attrs`` is a sequence - of additional attributes to be matched (specified as keyword arguments in the - class pattern), ``kwd_patterns`` are the corresponding patterns (specified as - keyword values in the class pattern). + A match class pattern. ``cls`` is an expression giving the nominal class to + be matched. ``patterns`` is a sequence of pattern nodes to be matched against + the class defined sequence of pattern matching attributes. ``kwd_attrs`` is a + sequence of additional attributes to be matched (specified as keyword arguments + in the class pattern), ``kwd_patterns`` are the corresponding patterns + (specified as keyword values in the class pattern). This pattern succeeds if the subject is an instance of the nominated class, all positional patterns match the corresponding class-defined attributes, and any specified keyword attributes match their corresponding pattern. + Note: classes may define a property that returns self in order to match a + pattern node against the instance being matched. Several builtin types are + also matched that way, as described in the match statement documentation. + .. doctest:: >>> print(ast.dump(ast.parse(""" @@ -1560,11 +1541,14 @@ Pattern matching .. class:: MatchAs(pattern, name) - A match "as-pattern" or capture pattern. ``pattern`` contains the match - pattern that will be used to match the subject against. If the pattern is - ``None``, the node represents a capture pattern (i.e a bare name) and will - always succeed. The ``name`` attribute contains the name that will be bound - if the pattern is successful. + A match "as-pattern", capture pattern or wildcard pattern. ``pattern`` + contains the match pattern that the subject will be matched against. + If the pattern is ``None``, the node represents a capture pattern (i.e a + bare name) and will always succeed. + + The ``name`` attribute contains the name that will be bound if the pattern + is successful. If ``name`` is ``None``, ``pattern`` must also be ``None`` + and the node represents the wildcard pattern. .. doctest:: @@ -1572,6 +1556,8 @@ Pattern matching ... match x: ... case [x] as y: ... ... + ... case _: + ... ... ... """), indent=4)) Module( body=[ @@ -1584,6 +1570,11 @@ Pattern matching patterns=[ MatchAs(name='x')]), name='y'), + body=[ + Expr( + value=Constant(value=Ellipsis))]), + match_case( + pattern=MatchAs(), body=[ Expr( value=Constant(value=Ellipsis))])])], @@ -1594,7 +1585,7 @@ Pattern matching A match "or-pattern". An or-pattern matches each of its subpatterns in turn to the subject, until one succeeds. The or-pattern is then deemed to succeed. If none of the subpatterns succeed the or-pattern fails. The - ``patterns`` attribute contains a list of match patterns nodes that will be + ``patterns`` attribute contains a list of match pattern nodes that will be matched against the subject. .. doctest:: diff --git a/Doc/myfile.bz2 b/Doc/myfile.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..7ada20f60926b48c5e828d1e7bf71413ab4e70e9 GIT binary patch literal 331 zcmV-R0kr-?T4*^jL0KkKS+!R|D*ym2SAYNzKm{!$Kmb4Y{{S!nQ)N(SCV(fDn4n-m zsj1*o)jw1{KzfI%pc>6yC8lH}rXzFA}n%<_UC%QtSrIv)E4R zDh^1A%zp>6B3vmTgeYY0oJp0(`BpUNh|T3{*-_`)^ao&) zc!V(so1042t!WwNmV`neq?w3@G%ivk5jbims09KR+7&i4b+DaUp9+qJE21@v>K~jq d&ZNWD>qxYMPv6On5RHF}xgwk>NLs6)m4J*en85%5 literal 0 HcmV?d00001 diff --git a/Grammar/python.gram b/Grammar/python.gram index d48ea0159c8482..ab73590de3df0a 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -291,7 +291,7 @@ pattern_capture_target[expr_ty]: _PyPegen_set_expr_context(p, name, Store) } wildcard_pattern[pattern_ty]: - | "_" { _PyAST_MatchAlways(EXTRA) } + | "_" { _PyAST_MatchAs(NULL, NULL, EXTRA) } value_pattern[pattern_ty]: | attr=attr !('.' | '(' | '=') { _PyAST_MatchValue(attr, EXTRA) } diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index e775a79708d456..ebb6a90087bb52 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -559,10 +559,10 @@ struct _match_case { asdl_stmt_seq *body; }; -enum _pattern_kind {MatchAlways_kind=1, MatchValue_kind=2, - MatchSingleton_kind=3, MatchSequence_kind=4, - MatchMapping_kind=5, MatchClass_kind=6, MatchStar_kind=7, - MatchAs_kind=8, MatchOr_kind=9}; +enum _pattern_kind {MatchValue_kind=1, MatchSingleton_kind=2, + MatchSequence_kind=3, MatchMapping_kind=4, + MatchClass_kind=5, MatchStar_kind=6, MatchAs_kind=7, + MatchOr_kind=8}; struct _pattern { enum _pattern_kind kind; union { @@ -808,8 +808,6 @@ withitem_ty _PyAST_withitem(expr_ty context_expr, expr_ty optional_vars, PyArena *arena); match_case_ty _PyAST_match_case(pattern_ty pattern, expr_ty guard, asdl_stmt_seq * body, PyArena *arena); -pattern_ty _PyAST_MatchAlways(int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); pattern_ty _PyAST_MatchValue(expr_ty value, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); pattern_ty _PyAST_MatchSingleton(constant value, int lineno, int col_offset, diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index e7c7b37d77b533..882cd09c00628d 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -91,7 +91,6 @@ struct ast_state { PyObject *Lt_type; PyObject *MatMult_singleton; PyObject *MatMult_type; - PyObject *MatchAlways_type; PyObject *MatchAs_type; PyObject *MatchClass_type; PyObject *MatchMapping_type; diff --git a/Lib/ast.py b/Lib/ast.py index ccb341ed06d0c1..f23c88fe5dbf18 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -1590,9 +1590,6 @@ def visit_match_case(self, node): with self.block(): self.traverse(node.body) - def visit_MatchAlways(self, node): - self.write("_") - def visit_MatchValue(self, node): self.traverse(node.value) @@ -1649,14 +1646,17 @@ def write_attr_pattern(pair): ) def visit_MatchAs(self, node): + name = node.name pattern = node.pattern - if pattern is not None: + if name is None: + self.write("_") + elif pattern is None: + self.write(node.name) + else: with self.require_parens(_Precedence.TEST, node): self.set_precedence(_Precedence.BOR, node.pattern) self.traverse(node.pattern) self.write(f" as {node.name}") - else: - self.write(node.name) def visit_MatchOr(self, node): with self.require_parens(_Precedence.BOR, node): diff --git a/Parser/Python.asdl b/Parser/Python.asdl index dec16bda5fce11..85225fc88c5a54 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -126,19 +126,19 @@ module Python match_case = (pattern pattern, expr? guard, stmt* body) - pattern = MatchAlways - | MatchValue(expr value) - | MatchSingleton(constant value) - | MatchSequence(pattern* patterns) - | MatchMapping(expr* keys, pattern* patterns, identifier? rest) - | MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns) - - | MatchStar(identifier? name) - -- The optional "rest" MatchMapping parameter handles capturing extra mapping keys - - | MatchAs(pattern? pattern, identifier name) - | MatchOr(pattern* patterns) - attributes (int lineno, int col_offset, int end_lineno, int end_col_offset) + pattern = MatchValue(expr value) + | MatchSingleton(constant value) + | MatchSequence(pattern* patterns) + | MatchMapping(expr* keys, pattern* patterns, identifier? rest) + | MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns) + + | MatchStar(identifier? name) + -- The optional "rest" MatchMapping parameter handles capturing extra mapping keys + + | MatchAs(pattern? pattern, identifier? name) + | MatchOr(pattern* patterns) + + attributes (int lineno, int col_offset, int end_lineno, int end_col_offset) type_ignore = TypeIgnore(int lineno, string tag) } diff --git a/Parser/parser.c b/Parser/parser.c index 04c3310e20479c..e468caf4636b52 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -6646,7 +6646,7 @@ wildcard_pattern_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_MatchAlways ( EXTRA ); + _res = _PyAST_MatchAs ( NULL , NULL , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); diff --git a/Python/Python-ast.c b/Python/Python-ast.c index d888538682d80b..5d7a0aed9beffc 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -105,7 +105,6 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(state->Lt_type); Py_CLEAR(state->MatMult_singleton); Py_CLEAR(state->MatMult_type); - Py_CLEAR(state->MatchAlways_type); Py_CLEAR(state->MatchAs_type); Py_CLEAR(state->MatchClass_type); Py_CLEAR(state->MatchMapping_type); @@ -1775,22 +1774,17 @@ init_types(struct ast_state *state) if (PyObject_SetAttr(state->match_case_type, state->guard, Py_None) == -1) return 0; state->pattern_type = make_type(state, "pattern", state->AST_type, NULL, 0, - "pattern = MatchAlways\n" - " | MatchValue(expr value)\n" + "pattern = MatchValue(expr value)\n" " | MatchSingleton(constant value)\n" " | MatchSequence(pattern* patterns)\n" " | MatchMapping(expr* keys, pattern* patterns, identifier? rest)\n" " | MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns)\n" " | MatchStar(identifier? name)\n" - " | MatchAs(pattern? pattern, identifier name)\n" + " | MatchAs(pattern? pattern, identifier? name)\n" " | MatchOr(pattern* patterns)"); if (!state->pattern_type) return 0; if (!add_attributes(state, state->pattern_type, pattern_attributes, 4)) return 0; - state->MatchAlways_type = make_type(state, "MatchAlways", - state->pattern_type, NULL, 0, - "MatchAlways"); - if (!state->MatchAlways_type) return 0; state->MatchValue_type = make_type(state, "MatchValue", state->pattern_type, MatchValue_fields, 1, @@ -1826,10 +1820,12 @@ init_types(struct ast_state *state) return 0; state->MatchAs_type = make_type(state, "MatchAs", state->pattern_type, MatchAs_fields, 2, - "MatchAs(pattern? pattern, identifier name)"); + "MatchAs(pattern? pattern, identifier? name)"); if (!state->MatchAs_type) return 0; if (PyObject_SetAttr(state->MatchAs_type, state->pattern, Py_None) == -1) return 0; + if (PyObject_SetAttr(state->MatchAs_type, state->name, Py_None) == -1) + return 0; state->MatchOr_type = make_type(state, "MatchOr", state->pattern_type, MatchOr_fields, 1, "MatchOr(pattern* patterns)"); @@ -3394,22 +3390,6 @@ _PyAST_match_case(pattern_ty pattern, expr_ty guard, asdl_stmt_seq * body, return p; } -pattern_ty -_PyAST_MatchAlways(int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena) -{ - pattern_ty p; - p = (pattern_ty)_PyArena_Malloc(arena, sizeof(*p)); - if (!p) - return NULL; - p->kind = MatchAlways_kind; - p->lineno = lineno; - p->col_offset = col_offset; - p->end_lineno = end_lineno; - p->end_col_offset = end_col_offset; - return p; -} - pattern_ty _PyAST_MatchValue(expr_ty value, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) @@ -3540,11 +3520,6 @@ _PyAST_MatchAs(pattern_ty pattern, identifier name, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) { pattern_ty p; - if (!name) { - PyErr_SetString(PyExc_ValueError, - "field 'name' is required for MatchAs"); - return NULL; - } p = (pattern_ty)_PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; @@ -5176,11 +5151,6 @@ ast2obj_pattern(struct ast_state *state, void* _o) Py_RETURN_NONE; } switch (o->kind) { - case MatchAlways_kind: - tp = (PyTypeObject *)state->MatchAlways_type; - result = PyType_GenericNew(tp, NULL, NULL); - if (!result) goto failed; - break; case MatchValue_kind: tp = (PyTypeObject *)state->MatchValue_type; result = PyType_GenericNew(tp, NULL, NULL); @@ -10319,18 +10289,6 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (res != 0) goto failed; Py_CLEAR(tmp); } - tp = state->MatchAlways_type; - isinstance = PyObject_IsInstance(obj, tp); - if (isinstance == -1) { - return 1; - } - if (isinstance) { - - *out = _PyAST_MatchAlways(lineno, col_offset, end_lineno, - end_col_offset, arena); - if (*out == NULL) goto failed; - return 0; - } tp = state->MatchValue_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { @@ -10703,9 +10661,9 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (_PyObject_LookupAttr(obj, state->name, &tmp) < 0) { return 1; } - if (tmp == NULL) { - PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from MatchAs"); - return 1; + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + name = NULL; } else { int res; @@ -11179,9 +11137,6 @@ astmodule_exec(PyObject *m) if (PyModule_AddObjectRef(m, "pattern", state->pattern_type) < 0) { return -1; } - if (PyModule_AddObjectRef(m, "MatchAlways", state->MatchAlways_type) < 0) { - return -1; - } if (PyModule_AddObjectRef(m, "MatchValue", state->MatchValue_type) < 0) { return -1; } diff --git a/Python/ast.c b/Python/ast.c index f5eefce6f924eb..25316fd3f5689e 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -490,10 +490,6 @@ validate_pattern(struct validator *state, pattern_ty p) // Coming soon: https://bugs.python.org/issue43897 (thanks Batuhan)! // TODO: Ensure no subnodes use "_" as an ordinary identifier switch (p->kind) { - case MatchAlways_kind: - // Nothing to check - ret = 1; - break; case MatchValue_kind: ret = validate_pattern_match_value(state, p->v.MatchValue.value); break; @@ -544,7 +540,14 @@ validate_pattern(struct validator *state, pattern_ty p) case MatchAs_kind: // TODO: check target name is valid if (p->v.MatchAs.pattern) { - ret = validate_pattern(state, p->v.MatchAs.pattern); + // If a pattern is given, the name must also be given + if (!p->v.MatchAs.name) { + PyErr_SetString(PyExc_ValueError, + "MatchAs must specify a target name if a pattern is given"); + return 0; + } else { + ret = validate_pattern(state, p->v.MatchAs.pattern); + } } else { ret = 1; } diff --git a/Python/ast_opt.c b/Python/ast_opt.c index d16b552addf303..254dd646c467ba 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -804,8 +804,6 @@ astfold_pattern(pattern_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) return 0; } switch (node_->kind) { - case MatchAlways_kind: - break; case MatchValue_kind: CALL(astfold_expr, expr_ty, node_->v.MatchValue.value); break; diff --git a/Python/compile.c b/Python/compile.c index cf4abf64ac1adc..5d7868c4b4ccf3 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -5597,7 +5597,7 @@ compiler_slice(struct compiler *c, expr_ty s) // jumping in the peephole optimizer than to detect or predict it here. #define WILDCARD_CHECK(N) \ - ((N)->kind == MatchAlways_kind) + ((N)->kind == MatchAs_kind && !(N)->v.MatchAs.name) #define WILDCARD_STAR_CHECK(N) \ ((N)->kind == MatchStar_kind && !(N)->v.MatchStar.name) @@ -5790,10 +5790,27 @@ compiler_pattern_capture(struct compiler *c, identifier n, pattern_context *pc) return 1; } +static int +compiler_pattern_wildcard(struct compiler *c, pattern_ty p, pattern_context *pc) +{ + assert(p->kind == MatchAs_kind); + if (!pc->allow_irrefutable) { + // Whoops, can't have a wildcard here! + const char *e = "wildcard makes remaining patterns unreachable"; + return compiler_error(c, e); + } + ADDOP(c, POP_TOP); + ADDOP_LOAD_CONST(c, Py_True); + return 1; +} + static int compiler_pattern_as(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchAs_kind); + if (p->v.MatchAs.name == NULL) { + return compiler_pattern_wildcard(c, p, pc); + } if (p->v.MatchAs.pattern == NULL) { if (!pc->allow_irrefutable) { // Whoops, can't have a name capture here! @@ -6174,20 +6191,6 @@ compiler_pattern_sequence(struct compiler *c, pattern_ty p, pattern_context *pc) return 1; } -static int -compiler_pattern_wildcard(struct compiler *c, pattern_ty p, pattern_context *pc) -{ - assert(p->kind == MatchAlways_kind); - if (!pc->allow_irrefutable) { - // Whoops, can't have a wildcard here! - const char *e = "wildcard makes remaining patterns unreachable"; - return compiler_error(c, e); - } - ADDOP(c, POP_TOP); - ADDOP_LOAD_CONST(c, Py_True); - return 1; -} - static int compiler_pattern_value(struct compiler *c, pattern_ty p, pattern_context *pc) { @@ -6216,8 +6219,6 @@ compiler_pattern(struct compiler *c, pattern_ty p, pattern_context *pc) { SET_LOC(c, p); switch (p->kind) { - case MatchAlways_kind: - return compiler_pattern_wildcard(c, p, pc); case MatchValue_kind: return compiler_pattern_value(c, p, pc); case MatchSingleton_kind: diff --git a/Python/symtable.c b/Python/symtable.c index 355bb8403f3902..e620f1ecc1bc2d 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1708,9 +1708,6 @@ symtable_visit_pattern(struct symtable *st, pattern_ty p) VISIT_QUIT(st, 0); } switch (p->kind) { - case MatchAlways_kind: - /* Nothing to do here. */ - break; case MatchValue_kind: VISIT(st, expr, p->v.MatchValue.value); break; @@ -1741,7 +1738,9 @@ symtable_visit_pattern(struct symtable *st, pattern_ty p) if (p->v.MatchAs.pattern) { VISIT(st, pattern, p->v.MatchAs.pattern); } - symtable_add_def(st, p->v.MatchAs.name, DEF_LOCAL); + if (p->v.MatchAs.name) { + symtable_add_def(st, p->v.MatchAs.name, DEF_LOCAL); + } break; case MatchOr_kind: VISIT_SEQ(st, pattern, p->v.MatchOr.patterns); From 4ed22ca2f1eb5508eb2c557a7915d953708e8e7c Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Tue, 27 Apr 2021 21:51:32 +1000 Subject: [PATCH 23/38] Remove and ignore local doctest execution artifacts --- Doc/.gitignore | 3 +++ Doc/example.ini | 13 ------------- Doc/myfile.bz2 | Bin 331 -> 0 bytes 3 files changed, 3 insertions(+), 13 deletions(-) create mode 100644 Doc/.gitignore delete mode 100644 Doc/example.ini delete mode 100644 Doc/myfile.bz2 diff --git a/Doc/.gitignore b/Doc/.gitignore new file mode 100644 index 00000000000000..97b972104f0404 --- /dev/null +++ b/Doc/.gitignore @@ -0,0 +1,3 @@ +# Ignore doctest artifacts +example.ini +myfile.bz2 diff --git a/Doc/example.ini b/Doc/example.ini deleted file mode 100644 index 69e147f5825889..00000000000000 --- a/Doc/example.ini +++ /dev/null @@ -1,13 +0,0 @@ -[DEFAULT] -serveraliveinterval = 45 -compression = yes -compressionlevel = 9 -forwardx11 = yes - -[bitbucket.org] -user = hg - -[topsecret.server.com] -port = 50022 -forwardx11 = no - diff --git a/Doc/myfile.bz2 b/Doc/myfile.bz2 deleted file mode 100644 index 7ada20f60926b48c5e828d1e7bf71413ab4e70e9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 331 zcmV-R0kr-?T4*^jL0KkKS+!R|D*ym2SAYNzKm{!$Kmb4Y{{S!nQ)N(SCV(fDn4n-m zsj1*o)jw1{KzfI%pc>6yC8lH}rXzFA}n%<_UC%QtSrIv)E4R zDh^1A%zp>6B3vmTgeYY0oJp0(`BpUNh|T3{*-_`)^ao&) zc!V(so1042t!WwNmV`neq?w3@G%ivk5jbims09KR+7&i4b+DaUp9+qJE21@v>K~jq d&ZNWD>qxYMPv6On5RHF}xgwk>NLs6)m4J*en85%5 From 8e5083c96e7bd8bf0719439076b33d4640899f19 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Tue, 27 Apr 2021 22:02:12 +1000 Subject: [PATCH 24/38] Fix trailing whitespace --- Doc/.gitignore | 4 +--- Doc/library/ast.rst | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Doc/.gitignore b/Doc/.gitignore index 97b972104f0404..751553b3acb938 100644 --- a/Doc/.gitignore +++ b/Doc/.gitignore @@ -1,3 +1 @@ -# Ignore doctest artifacts -example.ini -myfile.bz2 +*.bak diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index ba34e82e83453d..c7074c40f280c6 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -1545,7 +1545,7 @@ Pattern matching contains the match pattern that the subject will be matched against. If the pattern is ``None``, the node represents a capture pattern (i.e a bare name) and will always succeed. - + The ``name`` attribute contains the name that will be bound if the pattern is successful. If ``name`` is ``None``, ``pattern`` must also be ``None`` and the node represents the wildcard pattern. From 398929d7be5fe5969cf3e9a02c99cf3fccbe8e37 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 27 Apr 2021 11:34:19 -0700 Subject: [PATCH 25/38] Apply suggestions from code review --- Lib/ast.py | 12 +++++++++--- .../2021-04-25-08-35-11.bpo-43892.hr5Ke2.rst | 2 +- Python/ast.c | 19 +++++++++---------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/Lib/ast.py b/Lib/ast.py index f23c88fe5dbf18..0c53e5c5712f5e 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -1598,7 +1598,9 @@ def visit_MatchSingleton(self, node): def visit_MatchSequence(self, node): with self.delimit("[", "]"): - self.interleave(lambda: self.write(", "), self.traverse, node.patterns) + self.interleave( + lambda: self.write(", "), self.traverse, node.patterns + ) def visit_MatchStar(self, node): name = node.name @@ -1616,7 +1618,9 @@ def write_key_pattern_pair(pair): with self.delimit("{", "}"): keys = node.keys self.interleave( - lambda: self.write(", "), write_key_pattern_pair, zip(keys, node.patterns, strict=True) + lambda: self.write(", "), + write_key_pattern_pair, + zip(keys, node.patterns, strict=True), ) rest = node.rest if rest is not None: @@ -1642,7 +1646,9 @@ def write_attr_pattern(pair): if patterns: self.write(", ") self.interleave( - lambda: self.write(", "), write_attr_pattern, zip(attrs, node.kwd_patterns, strict=True) + lambda: self.write(", "), + write_attr_pattern, + zip(attrs, node.kwd_patterns, strict=True), ) def visit_MatchAs(self, node): diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-04-25-08-35-11.bpo-43892.hr5Ke2.rst b/Misc/NEWS.d/next/Core and Builtins/2021-04-25-08-35-11.bpo-43892.hr5Ke2.rst index 65997548df2138..69be1b243c7be5 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2021-04-25-08-35-11.bpo-43892.hr5Ke2.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2021-04-25-08-35-11.bpo-43892.hr5Ke2.rst @@ -1,4 +1,4 @@ -Match patterns now use new dedicated AST nodes (``MatchAlways``, ``MatchValue``, +Match patterns now use new dedicated AST nodes (``MatchValue``, ``MatchSingleton``, ``MatchSequence``, ``MatchStar``, ``MatchMapping``, ``MatchClass``) rather than reusing expression AST nodes. ``MatchAs`` and ``MatchOr`` are now defined as pattern nodes rather than as expression nodes. diff --git a/Python/ast.c b/Python/ast.c index 25316fd3f5689e..57d92a3824a2db 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -539,18 +539,17 @@ validate_pattern(struct validator *state, pattern_ty p) break; case MatchAs_kind: // TODO: check target name is valid - if (p->v.MatchAs.pattern) { - // If a pattern is given, the name must also be given - if (!p->v.MatchAs.name) { - PyErr_SetString(PyExc_ValueError, - "MatchAs must specify a target name if a pattern is given"); - return 0; - } else { - ret = validate_pattern(state, p->v.MatchAs.pattern); - } - } else { + if (p->v.MatchAs.pattern == NULL) { ret = 1; } + else if (p->v.MatchAs.name == NULL) { + PyErr_SetString(PyExc_ValueError, + "MatchAs must specify a target name if a pattern is given"); + return 0; + } + else { + ret = validate_pattern(state, p->v.MatchAs.pattern); + } break; case MatchOr_kind: // TODO: Validate all subpatterns From fe182d4054ecc5c78670361a7db360bcdbb6d3f5 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 27 Apr 2021 11:35:09 -0700 Subject: [PATCH 26/38] Delete Doc/.gitignore --- Doc/.gitignore | 1 - 1 file changed, 1 deletion(-) delete mode 100644 Doc/.gitignore diff --git a/Doc/.gitignore b/Doc/.gitignore deleted file mode 100644 index 751553b3acb938..00000000000000 --- a/Doc/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.bak From 2b076174297ce44333c9eda23b868ac07275ed1b Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 27 Apr 2021 11:36:04 -0700 Subject: [PATCH 27/38] Update assert --- Python/compile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/compile.c b/Python/compile.c index 5d7868c4b4ccf3..90c98c7f2caf59 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -5685,7 +5685,7 @@ pattern_helper_sequence_unpack(struct compiler *c, asdl_pattern_seq *patterns, } for (Py_ssize_t i = 0; i < size; i++) { pattern_ty pattern = asdl_seq_GET(patterns, i); - assert(i != star || pattern->kind == MatchStar_kind); + assert((i == star) == (pattern->kind == MatchStar_kind)); if (!compiler_pattern_subpattern(c, pattern, pc) || !compiler_addop_j(c, POP_JUMP_IF_FALSE, fails[i]) || compiler_next_block(c) == NULL) From b8add8ab9b849753dbc2500a559c0f49c092a54a Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 27 Apr 2021 11:36:23 -0700 Subject: [PATCH 28/38] Doon asdl_seq_LEN --- Python/compile.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index 90c98c7f2caf59..23f43ff8e5d901 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -5881,9 +5881,9 @@ compiler_pattern_class(struct compiler *c, pattern_ty p, pattern_context *pc) asdl_pattern_seq *patterns = p->v.MatchClass.patterns; asdl_identifier_seq *kwd_attrs = p->v.MatchClass.kwd_attrs; asdl_pattern_seq *kwd_patterns = p->v.MatchClass.kwd_patterns; - Py_ssize_t nargs = patterns ? asdl_seq_LEN(patterns) : 0; - Py_ssize_t nattrs = kwd_attrs ? asdl_seq_LEN(kwd_attrs) : 0; - Py_ssize_t nkwd_patterns = kwd_patterns ? asdl_seq_LEN(kwd_patterns) : 0; + Py_ssize_t nargs = asdl_seq_LEN(patterns); + Py_ssize_t nattrs = asdl_seq_LEN(kwd_attrs); + Py_ssize_t nkwd_patterns = asdl_seq_LEN(kwd_patterns); if (nattrs != nkwd_patterns) { // AST validator shouldn't let this happen, but if it does, // just fail, don't crash out of the interpreter From 490db51ec83cd03134541c8ba9f2af7da5675069 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 27 Apr 2021 13:56:55 -0700 Subject: [PATCH 29/38] Revert AST validation step --- Python/pythonrun.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 75af7ceb58949f..f00e3eb0de803f 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1366,7 +1366,7 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, return NULL; mod = _PyParser_ASTFromString(str, filename, start, flags, arena); - if (mod == NULL || !_PyAST_Validate(mod)) { + if (mod == NULL) { _PyArena_Free(arena); return NULL; } From ea4ea93e65484fefc52545b334e71fc4108bbfa0 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 27 Apr 2021 20:21:14 -0700 Subject: [PATCH 30/38] Reorder tests --- Lib/test/test_patma.py | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py index 68f8ba8c0daf0a..f3275529f22dfa 100644 --- a/Lib/test/test_patma.py +++ b/Lib/test/test_patma.py @@ -2429,17 +2429,6 @@ def test_patma_240(self): pass """) - @no_perf - def _disabled_test_patma_240b(self): - # Test disabled until bpo-43897 fleshes out the AST validator - # (the initial skeleton doesn't recurse into mapping patterns) - # Ensure mapping keys are also restricted - self.assert_syntax_error(""" - match ...: - case {0+0:_}: - pass - """) - @no_perf def test_patma_241(self): self.assert_syntax_error(""" @@ -2448,15 +2437,6 @@ def test_patma_241(self): pass """) - @no_perf - def test_patma_241b(self): - # Ensure mapping keys are also restricted - self.assert_syntax_error(""" - match ...: - case {f"":_}: - pass - """) - @no_perf def test_patma_242(self): self.assert_syntax_error(""" @@ -2877,6 +2857,22 @@ class Class: self.assertIs(y, None) self.assertIs(z, None) + @no_perf + def test_patma_283(self): + self.assert_syntax_error(""" + match ...: + case {0+0: _}: + pass + """) + + @no_perf + def test_patma_284(self): + self.assert_syntax_error(""" + match ...: + case {f"": _}: + pass + """) + class PerfPatma(TestPatma): From e4cc5d7f32a9b9dd7e611e8e8bfb23999f4a6f05 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 27 Apr 2021 20:21:58 -0700 Subject: [PATCH 31/38] Remove overly-aggressive optimizations --- Python/ast_opt.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 254dd646c467ba..037b147317b5be 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -805,7 +805,8 @@ astfold_pattern(pattern_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) } switch (node_->kind) { case MatchValue_kind: - CALL(astfold_expr, expr_ty, node_->v.MatchValue.value); + // Don't fold the value as an expression, since we need values like + // 0+0 to raise SyntaxErrors in the compiler! break; case MatchSingleton_kind: break; @@ -813,11 +814,12 @@ astfold_pattern(pattern_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) CALL_SEQ(astfold_pattern, pattern, node_->v.MatchSequence.patterns); break; case MatchMapping_kind: - CALL_SEQ(astfold_expr, expr, node_->v.MatchMapping.keys); + // Don't fold the keys as expressions, since we need keys like 0+0 + // to raise SyntaxErrors in the compiler! CALL_SEQ(astfold_pattern, pattern, node_->v.MatchMapping.patterns); break; case MatchClass_kind: - CALL(astfold_expr, expr_ty, node_->v.MatchClass.cls); + // No need to fold the class (it's just a dotted name). CALL_SEQ(astfold_pattern, pattern, node_->v.MatchClass.patterns); CALL_SEQ(astfold_pattern, pattern, node_->v.MatchClass.kwd_patterns); break; @@ -825,7 +827,7 @@ astfold_pattern(pattern_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) break; case MatchAs_kind: if (node_->v.MatchAs.pattern) { - CALL(astfold_pattern, expr_ty, node_->v.MatchAs.pattern); + CALL(astfold_pattern, pattern, node_->v.MatchAs.pattern); } break; case MatchOr_kind: From 4faea477deb32a81ba22951dcc48479710b941dd Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 27 Apr 2021 20:25:06 -0700 Subject: [PATCH 32/38] Numeric literals are validated in the compiler --- Python/ast.c | 112 +-------------------------------------------------- 1 file changed, 1 insertion(+), 111 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index 57d92a3824a2db..8c1b58ca825e16 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -368,116 +368,6 @@ validate_expr(struct validator *state, expr_ty exp, expr_context_ty ctx) } -// Note: the ensure_literal_* functions are only used to validate a restricted -// set of non-recursive literals that have already been checked with -// validate_expr, so they don't accept the validator state -static int -ensure_literal_number(expr_ty exp, bool allow_real, bool allow_imaginary) -{ - assert(exp->kind == Constant_kind); - PyObject *value = exp->v.Constant.value; - return (allow_real && PyFloat_CheckExact(value)) || - (allow_real && PyLong_CheckExact(value)) || - (allow_imaginary && PyComplex_CheckExact(value)); -} - -static int -ensure_literal_negative(expr_ty exp, bool allow_real, bool allow_imaginary) -{ - assert(exp->kind == UnaryOp_kind); - // Must be negation ... - if (exp->v.UnaryOp.op != USub) { - return 0; - } - // ... of a constant ... - expr_ty operand = exp->v.UnaryOp.operand; - if (operand->kind != Constant_kind) { - return 0; - } - // ... number - return ensure_literal_number(operand, allow_real, allow_imaginary); -} - -static int -ensure_literal_complex(expr_ty exp) -{ - assert(exp->kind == BinOp_kind); - expr_ty left = exp->v.BinOp.left; - expr_ty right = exp->v.BinOp.right; - // Ensure op is addition or subtraction - if (exp->v.BinOp.op != Add && exp->v.BinOp.op != Sub) { - return 0; - } - // Check LHS is a real number - switch (left->kind) - { - case Constant_kind: - if (!ensure_literal_number(left, /*real=*/true, /*imaginary=*/false)) { - return 0; - } - break; - case UnaryOp_kind: - if (!ensure_literal_negative(left, /*real=*/true, /*imaginary=*/false)) { - return 0; - } - break; - default: - return 0; - } - // Check RHS is an imaginary number - switch (right->kind) - { - case Constant_kind: - if (!ensure_literal_number(right, /*real=*/false, /*imaginary=*/true)) { - return 0; - } - break; - case UnaryOp_kind: - if (!ensure_literal_negative(right, /*real=*/false, /*imaginary=*/true)) { - return 0; - } - break; - default: - return 0; - } - return 1; -} - -static int -validate_pattern_match_value(struct validator *state, expr_ty exp) -{ - if (!validate_expr(state, exp, Load)) { - return 0; - } - - switch (exp->kind) - { - case Constant_kind: - case Attribute_kind: - // Constants and attribute lookups are always permitted - return 1; - case UnaryOp_kind: - // Negated numbers are permitted (whether real or imaginary) - // Compiler will complain if AST folding doesn't create a constant - if (ensure_literal_negative(exp, /*real=*/true, /*imaginary=*/true)) { - return 1; - } - break; - case BinOp_kind: - // Complex literals are permitted - // Compiler will complain if AST folding doesn't create a constant - if (ensure_literal_complex(exp)) { - return 1; - } - break; - default: - break; - } - PyErr_SetString(PyExc_SyntaxError, - "patterns may only match literals and attribute lookups"); - return 0; -} - static int validate_pattern(struct validator *state, pattern_ty p) { @@ -491,7 +381,7 @@ validate_pattern(struct validator *state, pattern_ty p) // TODO: Ensure no subnodes use "_" as an ordinary identifier switch (p->kind) { case MatchValue_kind: - ret = validate_pattern_match_value(state, p->v.MatchValue.value); + ret = validate_expr(state, p->v.MatchValue.value, Load); break; case MatchSingleton_kind: // TODO: Check constant is specifically None, True, or False From fe26e14c68baa930a7c73e509903bba57a87220f Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 27 Apr 2021 20:26:20 -0700 Subject: [PATCH 33/38] Validate numeric literals in the compiler --- Python/compile.c | 71 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 5 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index 23f43ff8e5d901..dd68870454c6a3 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -5602,9 +5602,69 @@ compiler_slice(struct compiler *c, expr_ty s) #define WILDCARD_STAR_CHECK(N) \ ((N)->kind == MatchStar_kind && !(N)->v.MatchStar.name) -// Limit permitted subexpressions, even if the parser & AST validator let them through -#define MATCH_VALUE_EXPR(N) \ - ((N)->kind == Constant_kind || (N)->kind == Attribute_kind) +#define CHECK_VALUE_COMPLEX(E) \ + ((E)->kind == BinOp_kind && \ + ((E)->v.BinOp.op == Add || (E)->v.BinOp.op == Sub) && \ + (CHECK_VALUE_REAL_POS((E)->v.BinOp.left) || \ + CHECK_VALUE_REAL_NEG((E)->v.BinOp.left)) && \ + CHECK_VALUE_IMAG_POS((E)->v.BinOp.right)) + +#define CHECK_VALUE_IMAG_NEG(E) \ + ((E)->kind == UnaryOp_kind && \ + (E)->v.UnaryOp.op == USub && \ + CHECK_VALUE_IMAG_POS((E)->v.UnaryOp.operand)) + +#define CHECK_VALUE_IMAG_POS(E) \ + ((E)->kind == Constant_kind && \ + PyComplex_CheckExact((E)->v.Constant.value) && \ + ((PyComplexObject*)(E)->v.Constant.value)->cval.real == 0 && \ + ((PyComplexObject*)(E)->v.Constant.value)->cval.imag >= 0) + +#define CHECK_VALUE_REAL_NEG(E) \ + ((E)->kind == UnaryOp_kind && \ + (E)->v.UnaryOp.op == USub && \ + CHECK_VALUE_REAL_POS((E)->v.UnaryOp.operand)) + +#define CHECK_VALUE_REAL_POS(E) \ + ((E)->kind == Constant_kind && \ + ((PyLong_CheckExact((E)->v.Constant.value) && \ + _PyLong_Sign((E)->v.Constant.value) >= 0) || \ + (PyFloat_CheckExact((E)->v.Constant.value) && \ + PyFloat_AS_DOUBLE((E)->v.Constant.value) >= 0))) + + +static int +check_value(expr_ty e, int allow_singleton) +{ + switch (e->kind) { + case Attribute_kind: + return 1; + case BinOp_kind: + return CHECK_VALUE_COMPLEX(e); + case Constant_kind: { + PyObject *v = e->v.Constant.value; + if (PyBytes_CheckExact(v) || PyUnicode_CheckExact(v)) { + return 1; + } + if (PyBool_Check(v) || v == Py_None) { + return allow_singleton; + } + return CHECK_VALUE_REAL_POS(e); + } + case UnaryOp_kind: + return CHECK_VALUE_REAL_NEG(e) || CHECK_VALUE_IMAG_NEG(e); + default: + return 0; + } + Py_UNREACHABLE(); +} + + +#undef CHECK_VALUE_COMPLEX +#undef CHECK_VALUE_IMAG_POS +#undef CHECK_VALUE_REAL_NEG +#undef CHECK_VALUE_REAL_POS + static int pattern_helper_store_name(struct compiler *c, identifier n, pattern_context *pc) @@ -6002,7 +6062,7 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, pattern_context *pc) c->u->u_col_offset = ((pattern_ty) asdl_seq_GET(patterns, i))->col_offset; return compiler_error(c, e); } - if (!MATCH_VALUE_EXPR(key)) { + if (!check_value(key, 1)) { const char *e = "mapping pattern keys may only match literals and attribute lookups"; return compiler_error(c, e); } @@ -6196,7 +6256,7 @@ compiler_pattern_value(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchValue_kind); expr_ty value = p->v.MatchValue.value; - if (!MATCH_VALUE_EXPR(value)) { + if (!check_value(value, 0)) { const char *e = "patterns may only match literals and attribute lookups"; return compiler_error(c, e); } @@ -6304,6 +6364,7 @@ compiler_match(struct compiler *c, stmt_ty s) } #undef WILDCARD_CHECK +#undef WILDCARD_STAR_CHECK /* End of the compiler section, beginning of the assembler section */ From ac154facd63f06aeeaccce494de78e4cc9e74e54 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Wed, 28 Apr 2021 22:09:32 +1000 Subject: [PATCH 34/38] Revert to checking subexpressions in the validator --- Python/ast.c | 112 ++++++++++++++++++++++++++++++++++++++++++++++- Python/compile.c | 70 +++-------------------------- 2 files changed, 116 insertions(+), 66 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index 8c1b58ca825e16..57d92a3824a2db 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -368,6 +368,116 @@ validate_expr(struct validator *state, expr_ty exp, expr_context_ty ctx) } +// Note: the ensure_literal_* functions are only used to validate a restricted +// set of non-recursive literals that have already been checked with +// validate_expr, so they don't accept the validator state +static int +ensure_literal_number(expr_ty exp, bool allow_real, bool allow_imaginary) +{ + assert(exp->kind == Constant_kind); + PyObject *value = exp->v.Constant.value; + return (allow_real && PyFloat_CheckExact(value)) || + (allow_real && PyLong_CheckExact(value)) || + (allow_imaginary && PyComplex_CheckExact(value)); +} + +static int +ensure_literal_negative(expr_ty exp, bool allow_real, bool allow_imaginary) +{ + assert(exp->kind == UnaryOp_kind); + // Must be negation ... + if (exp->v.UnaryOp.op != USub) { + return 0; + } + // ... of a constant ... + expr_ty operand = exp->v.UnaryOp.operand; + if (operand->kind != Constant_kind) { + return 0; + } + // ... number + return ensure_literal_number(operand, allow_real, allow_imaginary); +} + +static int +ensure_literal_complex(expr_ty exp) +{ + assert(exp->kind == BinOp_kind); + expr_ty left = exp->v.BinOp.left; + expr_ty right = exp->v.BinOp.right; + // Ensure op is addition or subtraction + if (exp->v.BinOp.op != Add && exp->v.BinOp.op != Sub) { + return 0; + } + // Check LHS is a real number + switch (left->kind) + { + case Constant_kind: + if (!ensure_literal_number(left, /*real=*/true, /*imaginary=*/false)) { + return 0; + } + break; + case UnaryOp_kind: + if (!ensure_literal_negative(left, /*real=*/true, /*imaginary=*/false)) { + return 0; + } + break; + default: + return 0; + } + // Check RHS is an imaginary number + switch (right->kind) + { + case Constant_kind: + if (!ensure_literal_number(right, /*real=*/false, /*imaginary=*/true)) { + return 0; + } + break; + case UnaryOp_kind: + if (!ensure_literal_negative(right, /*real=*/false, /*imaginary=*/true)) { + return 0; + } + break; + default: + return 0; + } + return 1; +} + +static int +validate_pattern_match_value(struct validator *state, expr_ty exp) +{ + if (!validate_expr(state, exp, Load)) { + return 0; + } + + switch (exp->kind) + { + case Constant_kind: + case Attribute_kind: + // Constants and attribute lookups are always permitted + return 1; + case UnaryOp_kind: + // Negated numbers are permitted (whether real or imaginary) + // Compiler will complain if AST folding doesn't create a constant + if (ensure_literal_negative(exp, /*real=*/true, /*imaginary=*/true)) { + return 1; + } + break; + case BinOp_kind: + // Complex literals are permitted + // Compiler will complain if AST folding doesn't create a constant + if (ensure_literal_complex(exp)) { + return 1; + } + break; + default: + break; + } + PyErr_SetString(PyExc_SyntaxError, + "patterns may only match literals and attribute lookups"); + return 0; +} + static int validate_pattern(struct validator *state, pattern_ty p) { @@ -381,7 +491,7 @@ validate_pattern(struct validator *state, pattern_ty p) // TODO: Ensure no subnodes use "_" as an ordinary identifier switch (p->kind) { case MatchValue_kind: - ret = validate_expr(state, p->v.MatchValue.value, Load); + ret = validate_pattern_match_value(state, p->v.MatchValue.value); break; case MatchSingleton_kind: // TODO: Check constant is specifically None, True, or False diff --git a/Python/compile.c b/Python/compile.c index dd68870454c6a3..3cf61221967593 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -5602,69 +5602,9 @@ compiler_slice(struct compiler *c, expr_ty s) #define WILDCARD_STAR_CHECK(N) \ ((N)->kind == MatchStar_kind && !(N)->v.MatchStar.name) -#define CHECK_VALUE_COMPLEX(E) \ - ((E)->kind == BinOp_kind && \ - ((E)->v.BinOp.op == Add || (E)->v.BinOp.op == Sub) && \ - (CHECK_VALUE_REAL_POS((E)->v.BinOp.left) || \ - CHECK_VALUE_REAL_NEG((E)->v.BinOp.left)) && \ - CHECK_VALUE_IMAG_POS((E)->v.BinOp.right)) - -#define CHECK_VALUE_IMAG_NEG(E) \ - ((E)->kind == UnaryOp_kind && \ - (E)->v.UnaryOp.op == USub && \ - CHECK_VALUE_IMAG_POS((E)->v.UnaryOp.operand)) - -#define CHECK_VALUE_IMAG_POS(E) \ - ((E)->kind == Constant_kind && \ - PyComplex_CheckExact((E)->v.Constant.value) && \ - ((PyComplexObject*)(E)->v.Constant.value)->cval.real == 0 && \ - ((PyComplexObject*)(E)->v.Constant.value)->cval.imag >= 0) - -#define CHECK_VALUE_REAL_NEG(E) \ - ((E)->kind == UnaryOp_kind && \ - (E)->v.UnaryOp.op == USub && \ - CHECK_VALUE_REAL_POS((E)->v.UnaryOp.operand)) - -#define CHECK_VALUE_REAL_POS(E) \ - ((E)->kind == Constant_kind && \ - ((PyLong_CheckExact((E)->v.Constant.value) && \ - _PyLong_Sign((E)->v.Constant.value) >= 0) || \ - (PyFloat_CheckExact((E)->v.Constant.value) && \ - PyFloat_AS_DOUBLE((E)->v.Constant.value) >= 0))) - - -static int -check_value(expr_ty e, int allow_singleton) -{ - switch (e->kind) { - case Attribute_kind: - return 1; - case BinOp_kind: - return CHECK_VALUE_COMPLEX(e); - case Constant_kind: { - PyObject *v = e->v.Constant.value; - if (PyBytes_CheckExact(v) || PyUnicode_CheckExact(v)) { - return 1; - } - if (PyBool_Check(v) || v == Py_None) { - return allow_singleton; - } - return CHECK_VALUE_REAL_POS(e); - } - case UnaryOp_kind: - return CHECK_VALUE_REAL_NEG(e) || CHECK_VALUE_IMAG_NEG(e); - default: - return 0; - } - Py_UNREACHABLE(); -} - - -#undef CHECK_VALUE_COMPLEX -#undef CHECK_VALUE_IMAG_POS -#undef CHECK_VALUE_REAL_NEG -#undef CHECK_VALUE_REAL_POS - +// Limit permitted subexpressions, even if the parser & AST validator let them through +#define MATCH_VALUE_EXPR(N) \ + ((N)->kind == Constant_kind || (N)->kind == Attribute_kind) static int pattern_helper_store_name(struct compiler *c, identifier n, pattern_context *pc) @@ -6062,7 +6002,7 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, pattern_context *pc) c->u->u_col_offset = ((pattern_ty) asdl_seq_GET(patterns, i))->col_offset; return compiler_error(c, e); } - if (!check_value(key, 1)) { + if (!MATCH_VALUE_EXPR(key)) { const char *e = "mapping pattern keys may only match literals and attribute lookups"; return compiler_error(c, e); } @@ -6256,7 +6196,7 @@ compiler_pattern_value(struct compiler *c, pattern_ty p, pattern_context *pc) { assert(p->kind == MatchValue_kind); expr_ty value = p->v.MatchValue.value; - if (!check_value(value, 0)) { + if (!MATCH_VALUE_EXPR(value)) { const char *e = "patterns may only match literals and attribute lookups"; return compiler_error(c, e); } From 3ef2880952e9d1d4df585e5d98a8ccb0f2785a55 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Wed, 28 Apr 2021 22:10:47 +1000 Subject: [PATCH 35/38] Ensure parser rejects 0+0 in match patterns --- Grammar/python.gram | 7 +- Include/internal/pycore_ast.h | 3 + Lib/test/test_patma.py | 9 + Parser/parser.c | 799 ++++++++++++++++++---------------- Parser/pegen.c | 2 +- Python/ast.c | 19 +- 6 files changed, 451 insertions(+), 388 deletions(-) diff --git a/Grammar/python.gram b/Grammar/python.gram index ab73590de3df0a..5fc8f3e999996d 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -276,13 +276,16 @@ literal_expr[expr_ty]: | 'False' { _PyAST_Constant(Py_False, NULL, EXTRA) } complex_number[expr_ty]: - | real=signed_number '+' imag=NUMBER { _PyAST_BinOp(real, Add, imag, EXTRA) } - | real=signed_number '-' imag=NUMBER { _PyAST_BinOp(real, Sub, imag, EXTRA) } + | real=signed_number '+' imag=imaginary_number { _PyAST_BinOp(real, Add, imag, EXTRA) } + | real=signed_number '-' imag=imaginary_number { _PyAST_BinOp(real, Sub, imag, EXTRA) } signed_number[expr_ty]: | NUMBER | '-' number=NUMBER { _PyAST_UnaryOp(USub, number, EXTRA) } +imaginary_number[expr_ty]: + | imag=NUMBER { _PyAST_EnsureImaginary(imag) } + capture_pattern[pattern_ty]: | target=pattern_capture_target { _PyAST_MatchAs(NULL, target->v.Name.id, EXTRA) } diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index ebb6a90087bb52..05fca96c75f61f 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -841,6 +841,9 @@ int PyAST_Check(PyObject* obj); extern int _PyAST_Validate(mod_ty); +/* Helper to let the parser reject 0+0 (et al) when it wants a complex literal */ +extern expr_ty _PyAST_EnsureImaginary(expr_ty); + /* _PyAST_ExprAsUnicode is defined in ast_unparse.c */ extern PyObject* _PyAST_ExprAsUnicode(expr_ty); diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py index f3275529f22dfa..4371bcc3337ffb 100644 --- a/Lib/test/test_patma.py +++ b/Lib/test/test_patma.py @@ -2429,6 +2429,15 @@ def test_patma_240(self): pass """) + @no_perf + def test_patma_240b(self): + # Ensure mapping keys are also restricted + self.assert_syntax_error(""" + match ...: + case {0+0:_}: + pass + """) + @no_perf def test_patma_241(self): self.assert_syntax_error(""" diff --git a/Parser/parser.c b/Parser/parser.c index e468caf4636b52..c52da7d2f262c2 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -122,373 +122,374 @@ static char *soft_keywords[] = { #define literal_expr_type 1048 #define complex_number_type 1049 #define signed_number_type 1050 -#define capture_pattern_type 1051 -#define pattern_capture_target_type 1052 -#define wildcard_pattern_type 1053 -#define value_pattern_type 1054 -#define attr_type 1055 // Left-recursive -#define name_or_attr_type 1056 // Left-recursive -#define group_pattern_type 1057 -#define sequence_pattern_type 1058 -#define open_sequence_pattern_type 1059 -#define maybe_sequence_pattern_type 1060 -#define maybe_star_pattern_type 1061 -#define star_pattern_type 1062 -#define mapping_pattern_type 1063 -#define items_pattern_type 1064 -#define key_value_pattern_type 1065 -#define double_star_pattern_type 1066 -#define class_pattern_type 1067 -#define positional_patterns_type 1068 -#define keyword_patterns_type 1069 -#define keyword_pattern_type 1070 -#define return_stmt_type 1071 -#define raise_stmt_type 1072 -#define function_def_type 1073 -#define function_def_raw_type 1074 -#define func_type_comment_type 1075 -#define params_type 1076 -#define parameters_type 1077 -#define slash_no_default_type 1078 -#define slash_with_default_type 1079 -#define star_etc_type 1080 -#define kwds_type 1081 -#define param_no_default_type 1082 -#define param_with_default_type 1083 -#define param_maybe_default_type 1084 -#define param_type 1085 -#define annotation_type 1086 -#define default_type 1087 -#define decorators_type 1088 -#define class_def_type 1089 -#define class_def_raw_type 1090 -#define block_type 1091 -#define star_expressions_type 1092 -#define star_expression_type 1093 -#define star_named_expressions_type 1094 -#define star_named_expression_type 1095 -#define named_expression_type 1096 -#define direct_named_expression_type 1097 -#define annotated_rhs_type 1098 -#define expressions_type 1099 -#define expression_type 1100 -#define lambdef_type 1101 -#define lambda_params_type 1102 -#define lambda_parameters_type 1103 -#define lambda_slash_no_default_type 1104 -#define lambda_slash_with_default_type 1105 -#define lambda_star_etc_type 1106 -#define lambda_kwds_type 1107 -#define lambda_param_no_default_type 1108 -#define lambda_param_with_default_type 1109 -#define lambda_param_maybe_default_type 1110 -#define lambda_param_type 1111 -#define disjunction_type 1112 -#define conjunction_type 1113 -#define inversion_type 1114 -#define comparison_type 1115 -#define compare_op_bitwise_or_pair_type 1116 -#define eq_bitwise_or_type 1117 -#define noteq_bitwise_or_type 1118 -#define lte_bitwise_or_type 1119 -#define lt_bitwise_or_type 1120 -#define gte_bitwise_or_type 1121 -#define gt_bitwise_or_type 1122 -#define notin_bitwise_or_type 1123 -#define in_bitwise_or_type 1124 -#define isnot_bitwise_or_type 1125 -#define is_bitwise_or_type 1126 -#define bitwise_or_type 1127 // Left-recursive -#define bitwise_xor_type 1128 // Left-recursive -#define bitwise_and_type 1129 // Left-recursive -#define shift_expr_type 1130 // Left-recursive -#define sum_type 1131 // Left-recursive -#define term_type 1132 // Left-recursive -#define factor_type 1133 -#define power_type 1134 -#define await_primary_type 1135 -#define primary_type 1136 // Left-recursive -#define slices_type 1137 -#define slice_type 1138 -#define atom_type 1139 -#define strings_type 1140 -#define list_type 1141 -#define listcomp_type 1142 -#define tuple_type 1143 -#define group_type 1144 -#define genexp_type 1145 -#define set_type 1146 -#define setcomp_type 1147 -#define dict_type 1148 -#define dictcomp_type 1149 -#define double_starred_kvpairs_type 1150 -#define double_starred_kvpair_type 1151 -#define kvpair_type 1152 -#define for_if_clauses_type 1153 -#define for_if_clause_type 1154 -#define yield_expr_type 1155 -#define arguments_type 1156 -#define args_type 1157 -#define kwargs_type 1158 -#define starred_expression_type 1159 -#define kwarg_or_starred_type 1160 -#define kwarg_or_double_starred_type 1161 -#define star_targets_type 1162 -#define star_targets_list_seq_type 1163 -#define star_targets_tuple_seq_type 1164 -#define star_target_type 1165 -#define target_with_star_atom_type 1166 -#define star_atom_type 1167 -#define single_target_type 1168 -#define single_subscript_attribute_target_type 1169 -#define del_targets_type 1170 -#define del_target_type 1171 -#define del_t_atom_type 1172 -#define targets_type 1173 -#define target_type 1174 -#define t_primary_type 1175 // Left-recursive -#define t_lookahead_type 1176 -#define t_atom_type 1177 -#define invalid_arguments_type 1178 -#define invalid_kwarg_type 1179 -#define invalid_expression_type 1180 -#define invalid_named_expression_type 1181 -#define invalid_assignment_type 1182 -#define invalid_ann_assign_target_type 1183 -#define invalid_del_stmt_type 1184 -#define invalid_block_type 1185 -#define invalid_primary_type 1186 // Left-recursive -#define invalid_comprehension_type 1187 -#define invalid_dict_comprehension_type 1188 -#define invalid_parameters_type 1189 -#define invalid_parameters_helper_type 1190 -#define invalid_lambda_parameters_type 1191 -#define invalid_lambda_parameters_helper_type 1192 -#define invalid_star_etc_type 1193 -#define invalid_lambda_star_etc_type 1194 -#define invalid_double_type_comments_type 1195 -#define invalid_with_item_type 1196 -#define invalid_for_target_type 1197 -#define invalid_group_type 1198 -#define invalid_import_from_targets_type 1199 -#define invalid_with_stmt_type 1200 -#define invalid_with_stmt_indent_type 1201 -#define invalid_try_stmt_type 1202 -#define invalid_except_stmt_type 1203 -#define invalid_finally_stmt_type 1204 -#define invalid_except_stmt_indent_type 1205 -#define invalid_match_stmt_type 1206 -#define invalid_case_block_type 1207 -#define invalid_if_stmt_type 1208 -#define invalid_elif_stmt_type 1209 -#define invalid_else_stmt_type 1210 -#define invalid_while_stmt_type 1211 -#define invalid_for_stmt_type 1212 -#define invalid_def_raw_type 1213 -#define invalid_class_def_raw_type 1214 -#define invalid_double_starred_kvpairs_type 1215 -#define invalid_kvpair_type 1216 -#define _loop0_1_type 1217 -#define _loop0_2_type 1218 -#define _loop0_4_type 1219 -#define _gather_3_type 1220 -#define _loop0_6_type 1221 -#define _gather_5_type 1222 -#define _loop0_8_type 1223 -#define _gather_7_type 1224 -#define _loop0_10_type 1225 -#define _gather_9_type 1226 -#define _loop1_11_type 1227 -#define _loop0_13_type 1228 -#define _gather_12_type 1229 -#define _tmp_14_type 1230 -#define _tmp_15_type 1231 -#define _tmp_16_type 1232 -#define _tmp_17_type 1233 -#define _tmp_18_type 1234 -#define _tmp_19_type 1235 -#define _tmp_20_type 1236 -#define _tmp_21_type 1237 -#define _loop1_22_type 1238 -#define _tmp_23_type 1239 -#define _tmp_24_type 1240 -#define _loop0_26_type 1241 -#define _gather_25_type 1242 -#define _loop0_28_type 1243 -#define _gather_27_type 1244 -#define _tmp_29_type 1245 -#define _tmp_30_type 1246 -#define _loop0_31_type 1247 -#define _loop1_32_type 1248 -#define _loop0_34_type 1249 -#define _gather_33_type 1250 -#define _tmp_35_type 1251 -#define _loop0_37_type 1252 -#define _gather_36_type 1253 -#define _tmp_38_type 1254 -#define _loop0_40_type 1255 -#define _gather_39_type 1256 -#define _loop0_42_type 1257 -#define _gather_41_type 1258 -#define _loop0_44_type 1259 -#define _gather_43_type 1260 -#define _loop0_46_type 1261 -#define _gather_45_type 1262 -#define _tmp_47_type 1263 -#define _loop1_48_type 1264 -#define _tmp_49_type 1265 -#define _loop1_50_type 1266 -#define _loop0_52_type 1267 -#define _gather_51_type 1268 -#define _tmp_53_type 1269 -#define _tmp_54_type 1270 -#define _tmp_55_type 1271 -#define _tmp_56_type 1272 -#define _loop0_58_type 1273 -#define _gather_57_type 1274 -#define _loop0_60_type 1275 -#define _gather_59_type 1276 -#define _tmp_61_type 1277 -#define _loop0_63_type 1278 -#define _gather_62_type 1279 -#define _loop0_65_type 1280 -#define _gather_64_type 1281 -#define _tmp_66_type 1282 -#define _tmp_67_type 1283 -#define _tmp_68_type 1284 -#define _tmp_69_type 1285 -#define _loop0_70_type 1286 -#define _loop0_71_type 1287 -#define _loop0_72_type 1288 -#define _loop1_73_type 1289 -#define _loop0_74_type 1290 -#define _loop1_75_type 1291 -#define _loop1_76_type 1292 -#define _loop1_77_type 1293 -#define _loop0_78_type 1294 -#define _loop1_79_type 1295 -#define _loop0_80_type 1296 -#define _loop1_81_type 1297 -#define _loop0_82_type 1298 -#define _loop1_83_type 1299 -#define _loop1_84_type 1300 -#define _tmp_85_type 1301 -#define _loop1_86_type 1302 -#define _loop0_88_type 1303 -#define _gather_87_type 1304 -#define _loop1_89_type 1305 -#define _loop0_90_type 1306 -#define _loop0_91_type 1307 -#define _loop0_92_type 1308 -#define _loop1_93_type 1309 -#define _loop0_94_type 1310 -#define _loop1_95_type 1311 -#define _loop1_96_type 1312 -#define _loop1_97_type 1313 -#define _loop0_98_type 1314 -#define _loop1_99_type 1315 -#define _loop0_100_type 1316 -#define _loop1_101_type 1317 -#define _loop0_102_type 1318 -#define _loop1_103_type 1319 -#define _loop1_104_type 1320 -#define _loop1_105_type 1321 -#define _loop1_106_type 1322 -#define _tmp_107_type 1323 -#define _loop0_109_type 1324 -#define _gather_108_type 1325 -#define _tmp_110_type 1326 -#define _tmp_111_type 1327 -#define _tmp_112_type 1328 -#define _tmp_113_type 1329 -#define _loop1_114_type 1330 -#define _tmp_115_type 1331 -#define _tmp_116_type 1332 -#define _loop0_118_type 1333 -#define _gather_117_type 1334 -#define _loop1_119_type 1335 -#define _loop0_120_type 1336 -#define _loop0_121_type 1337 -#define _loop0_123_type 1338 -#define _gather_122_type 1339 -#define _tmp_124_type 1340 -#define _loop0_126_type 1341 -#define _gather_125_type 1342 -#define _loop0_128_type 1343 -#define _gather_127_type 1344 -#define _loop0_130_type 1345 -#define _gather_129_type 1346 -#define _loop0_132_type 1347 -#define _gather_131_type 1348 -#define _loop0_133_type 1349 -#define _loop0_135_type 1350 -#define _gather_134_type 1351 -#define _loop1_136_type 1352 -#define _tmp_137_type 1353 -#define _loop0_139_type 1354 -#define _gather_138_type 1355 -#define _loop0_141_type 1356 -#define _gather_140_type 1357 -#define _tmp_142_type 1358 -#define _tmp_143_type 1359 -#define _tmp_144_type 1360 -#define _tmp_145_type 1361 -#define _tmp_146_type 1362 -#define _loop0_147_type 1363 -#define _loop0_148_type 1364 -#define _loop0_149_type 1365 -#define _tmp_150_type 1366 -#define _tmp_151_type 1367 -#define _tmp_152_type 1368 -#define _tmp_153_type 1369 -#define _loop0_154_type 1370 -#define _loop1_155_type 1371 -#define _loop0_156_type 1372 -#define _loop1_157_type 1373 -#define _tmp_158_type 1374 -#define _tmp_159_type 1375 -#define _tmp_160_type 1376 -#define _loop0_162_type 1377 -#define _gather_161_type 1378 -#define _loop0_164_type 1379 -#define _gather_163_type 1380 -#define _loop0_166_type 1381 -#define _gather_165_type 1382 -#define _loop0_168_type 1383 -#define _gather_167_type 1384 -#define _tmp_169_type 1385 -#define _tmp_170_type 1386 -#define _tmp_171_type 1387 -#define _tmp_172_type 1388 -#define _tmp_173_type 1389 -#define _loop0_175_type 1390 -#define _gather_174_type 1391 -#define _tmp_176_type 1392 -#define _tmp_177_type 1393 -#define _tmp_178_type 1394 -#define _tmp_179_type 1395 -#define _tmp_180_type 1396 -#define _tmp_181_type 1397 -#define _tmp_182_type 1398 -#define _tmp_183_type 1399 -#define _tmp_184_type 1400 -#define _tmp_185_type 1401 -#define _tmp_186_type 1402 -#define _tmp_187_type 1403 -#define _tmp_188_type 1404 -#define _tmp_189_type 1405 -#define _tmp_190_type 1406 -#define _tmp_191_type 1407 -#define _tmp_192_type 1408 -#define _tmp_193_type 1409 -#define _tmp_194_type 1410 -#define _tmp_195_type 1411 -#define _tmp_196_type 1412 -#define _tmp_197_type 1413 -#define _tmp_198_type 1414 -#define _tmp_199_type 1415 -#define _tmp_200_type 1416 -#define _tmp_201_type 1417 +#define imaginary_number_type 1051 +#define capture_pattern_type 1052 +#define pattern_capture_target_type 1053 +#define wildcard_pattern_type 1054 +#define value_pattern_type 1055 +#define attr_type 1056 // Left-recursive +#define name_or_attr_type 1057 // Left-recursive +#define group_pattern_type 1058 +#define sequence_pattern_type 1059 +#define open_sequence_pattern_type 1060 +#define maybe_sequence_pattern_type 1061 +#define maybe_star_pattern_type 1062 +#define star_pattern_type 1063 +#define mapping_pattern_type 1064 +#define items_pattern_type 1065 +#define key_value_pattern_type 1066 +#define double_star_pattern_type 1067 +#define class_pattern_type 1068 +#define positional_patterns_type 1069 +#define keyword_patterns_type 1070 +#define keyword_pattern_type 1071 +#define return_stmt_type 1072 +#define raise_stmt_type 1073 +#define function_def_type 1074 +#define function_def_raw_type 1075 +#define func_type_comment_type 1076 +#define params_type 1077 +#define parameters_type 1078 +#define slash_no_default_type 1079 +#define slash_with_default_type 1080 +#define star_etc_type 1081 +#define kwds_type 1082 +#define param_no_default_type 1083 +#define param_with_default_type 1084 +#define param_maybe_default_type 1085 +#define param_type 1086 +#define annotation_type 1087 +#define default_type 1088 +#define decorators_type 1089 +#define class_def_type 1090 +#define class_def_raw_type 1091 +#define block_type 1092 +#define star_expressions_type 1093 +#define star_expression_type 1094 +#define star_named_expressions_type 1095 +#define star_named_expression_type 1096 +#define named_expression_type 1097 +#define direct_named_expression_type 1098 +#define annotated_rhs_type 1099 +#define expressions_type 1100 +#define expression_type 1101 +#define lambdef_type 1102 +#define lambda_params_type 1103 +#define lambda_parameters_type 1104 +#define lambda_slash_no_default_type 1105 +#define lambda_slash_with_default_type 1106 +#define lambda_star_etc_type 1107 +#define lambda_kwds_type 1108 +#define lambda_param_no_default_type 1109 +#define lambda_param_with_default_type 1110 +#define lambda_param_maybe_default_type 1111 +#define lambda_param_type 1112 +#define disjunction_type 1113 +#define conjunction_type 1114 +#define inversion_type 1115 +#define comparison_type 1116 +#define compare_op_bitwise_or_pair_type 1117 +#define eq_bitwise_or_type 1118 +#define noteq_bitwise_or_type 1119 +#define lte_bitwise_or_type 1120 +#define lt_bitwise_or_type 1121 +#define gte_bitwise_or_type 1122 +#define gt_bitwise_or_type 1123 +#define notin_bitwise_or_type 1124 +#define in_bitwise_or_type 1125 +#define isnot_bitwise_or_type 1126 +#define is_bitwise_or_type 1127 +#define bitwise_or_type 1128 // Left-recursive +#define bitwise_xor_type 1129 // Left-recursive +#define bitwise_and_type 1130 // Left-recursive +#define shift_expr_type 1131 // Left-recursive +#define sum_type 1132 // Left-recursive +#define term_type 1133 // Left-recursive +#define factor_type 1134 +#define power_type 1135 +#define await_primary_type 1136 +#define primary_type 1137 // Left-recursive +#define slices_type 1138 +#define slice_type 1139 +#define atom_type 1140 +#define strings_type 1141 +#define list_type 1142 +#define listcomp_type 1143 +#define tuple_type 1144 +#define group_type 1145 +#define genexp_type 1146 +#define set_type 1147 +#define setcomp_type 1148 +#define dict_type 1149 +#define dictcomp_type 1150 +#define double_starred_kvpairs_type 1151 +#define double_starred_kvpair_type 1152 +#define kvpair_type 1153 +#define for_if_clauses_type 1154 +#define for_if_clause_type 1155 +#define yield_expr_type 1156 +#define arguments_type 1157 +#define args_type 1158 +#define kwargs_type 1159 +#define starred_expression_type 1160 +#define kwarg_or_starred_type 1161 +#define kwarg_or_double_starred_type 1162 +#define star_targets_type 1163 +#define star_targets_list_seq_type 1164 +#define star_targets_tuple_seq_type 1165 +#define star_target_type 1166 +#define target_with_star_atom_type 1167 +#define star_atom_type 1168 +#define single_target_type 1169 +#define single_subscript_attribute_target_type 1170 +#define del_targets_type 1171 +#define del_target_type 1172 +#define del_t_atom_type 1173 +#define targets_type 1174 +#define target_type 1175 +#define t_primary_type 1176 // Left-recursive +#define t_lookahead_type 1177 +#define t_atom_type 1178 +#define invalid_arguments_type 1179 +#define invalid_kwarg_type 1180 +#define invalid_expression_type 1181 +#define invalid_named_expression_type 1182 +#define invalid_assignment_type 1183 +#define invalid_ann_assign_target_type 1184 +#define invalid_del_stmt_type 1185 +#define invalid_block_type 1186 +#define invalid_primary_type 1187 // Left-recursive +#define invalid_comprehension_type 1188 +#define invalid_dict_comprehension_type 1189 +#define invalid_parameters_type 1190 +#define invalid_parameters_helper_type 1191 +#define invalid_lambda_parameters_type 1192 +#define invalid_lambda_parameters_helper_type 1193 +#define invalid_star_etc_type 1194 +#define invalid_lambda_star_etc_type 1195 +#define invalid_double_type_comments_type 1196 +#define invalid_with_item_type 1197 +#define invalid_for_target_type 1198 +#define invalid_group_type 1199 +#define invalid_import_from_targets_type 1200 +#define invalid_with_stmt_type 1201 +#define invalid_with_stmt_indent_type 1202 +#define invalid_try_stmt_type 1203 +#define invalid_except_stmt_type 1204 +#define invalid_finally_stmt_type 1205 +#define invalid_except_stmt_indent_type 1206 +#define invalid_match_stmt_type 1207 +#define invalid_case_block_type 1208 +#define invalid_if_stmt_type 1209 +#define invalid_elif_stmt_type 1210 +#define invalid_else_stmt_type 1211 +#define invalid_while_stmt_type 1212 +#define invalid_for_stmt_type 1213 +#define invalid_def_raw_type 1214 +#define invalid_class_def_raw_type 1215 +#define invalid_double_starred_kvpairs_type 1216 +#define invalid_kvpair_type 1217 +#define _loop0_1_type 1218 +#define _loop0_2_type 1219 +#define _loop0_4_type 1220 +#define _gather_3_type 1221 +#define _loop0_6_type 1222 +#define _gather_5_type 1223 +#define _loop0_8_type 1224 +#define _gather_7_type 1225 +#define _loop0_10_type 1226 +#define _gather_9_type 1227 +#define _loop1_11_type 1228 +#define _loop0_13_type 1229 +#define _gather_12_type 1230 +#define _tmp_14_type 1231 +#define _tmp_15_type 1232 +#define _tmp_16_type 1233 +#define _tmp_17_type 1234 +#define _tmp_18_type 1235 +#define _tmp_19_type 1236 +#define _tmp_20_type 1237 +#define _tmp_21_type 1238 +#define _loop1_22_type 1239 +#define _tmp_23_type 1240 +#define _tmp_24_type 1241 +#define _loop0_26_type 1242 +#define _gather_25_type 1243 +#define _loop0_28_type 1244 +#define _gather_27_type 1245 +#define _tmp_29_type 1246 +#define _tmp_30_type 1247 +#define _loop0_31_type 1248 +#define _loop1_32_type 1249 +#define _loop0_34_type 1250 +#define _gather_33_type 1251 +#define _tmp_35_type 1252 +#define _loop0_37_type 1253 +#define _gather_36_type 1254 +#define _tmp_38_type 1255 +#define _loop0_40_type 1256 +#define _gather_39_type 1257 +#define _loop0_42_type 1258 +#define _gather_41_type 1259 +#define _loop0_44_type 1260 +#define _gather_43_type 1261 +#define _loop0_46_type 1262 +#define _gather_45_type 1263 +#define _tmp_47_type 1264 +#define _loop1_48_type 1265 +#define _tmp_49_type 1266 +#define _loop1_50_type 1267 +#define _loop0_52_type 1268 +#define _gather_51_type 1269 +#define _tmp_53_type 1270 +#define _tmp_54_type 1271 +#define _tmp_55_type 1272 +#define _tmp_56_type 1273 +#define _loop0_58_type 1274 +#define _gather_57_type 1275 +#define _loop0_60_type 1276 +#define _gather_59_type 1277 +#define _tmp_61_type 1278 +#define _loop0_63_type 1279 +#define _gather_62_type 1280 +#define _loop0_65_type 1281 +#define _gather_64_type 1282 +#define _tmp_66_type 1283 +#define _tmp_67_type 1284 +#define _tmp_68_type 1285 +#define _tmp_69_type 1286 +#define _loop0_70_type 1287 +#define _loop0_71_type 1288 +#define _loop0_72_type 1289 +#define _loop1_73_type 1290 +#define _loop0_74_type 1291 +#define _loop1_75_type 1292 +#define _loop1_76_type 1293 +#define _loop1_77_type 1294 +#define _loop0_78_type 1295 +#define _loop1_79_type 1296 +#define _loop0_80_type 1297 +#define _loop1_81_type 1298 +#define _loop0_82_type 1299 +#define _loop1_83_type 1300 +#define _loop1_84_type 1301 +#define _tmp_85_type 1302 +#define _loop1_86_type 1303 +#define _loop0_88_type 1304 +#define _gather_87_type 1305 +#define _loop1_89_type 1306 +#define _loop0_90_type 1307 +#define _loop0_91_type 1308 +#define _loop0_92_type 1309 +#define _loop1_93_type 1310 +#define _loop0_94_type 1311 +#define _loop1_95_type 1312 +#define _loop1_96_type 1313 +#define _loop1_97_type 1314 +#define _loop0_98_type 1315 +#define _loop1_99_type 1316 +#define _loop0_100_type 1317 +#define _loop1_101_type 1318 +#define _loop0_102_type 1319 +#define _loop1_103_type 1320 +#define _loop1_104_type 1321 +#define _loop1_105_type 1322 +#define _loop1_106_type 1323 +#define _tmp_107_type 1324 +#define _loop0_109_type 1325 +#define _gather_108_type 1326 +#define _tmp_110_type 1327 +#define _tmp_111_type 1328 +#define _tmp_112_type 1329 +#define _tmp_113_type 1330 +#define _loop1_114_type 1331 +#define _tmp_115_type 1332 +#define _tmp_116_type 1333 +#define _loop0_118_type 1334 +#define _gather_117_type 1335 +#define _loop1_119_type 1336 +#define _loop0_120_type 1337 +#define _loop0_121_type 1338 +#define _loop0_123_type 1339 +#define _gather_122_type 1340 +#define _tmp_124_type 1341 +#define _loop0_126_type 1342 +#define _gather_125_type 1343 +#define _loop0_128_type 1344 +#define _gather_127_type 1345 +#define _loop0_130_type 1346 +#define _gather_129_type 1347 +#define _loop0_132_type 1348 +#define _gather_131_type 1349 +#define _loop0_133_type 1350 +#define _loop0_135_type 1351 +#define _gather_134_type 1352 +#define _loop1_136_type 1353 +#define _tmp_137_type 1354 +#define _loop0_139_type 1355 +#define _gather_138_type 1356 +#define _loop0_141_type 1357 +#define _gather_140_type 1358 +#define _tmp_142_type 1359 +#define _tmp_143_type 1360 +#define _tmp_144_type 1361 +#define _tmp_145_type 1362 +#define _tmp_146_type 1363 +#define _loop0_147_type 1364 +#define _loop0_148_type 1365 +#define _loop0_149_type 1366 +#define _tmp_150_type 1367 +#define _tmp_151_type 1368 +#define _tmp_152_type 1369 +#define _tmp_153_type 1370 +#define _loop0_154_type 1371 +#define _loop1_155_type 1372 +#define _loop0_156_type 1373 +#define _loop1_157_type 1374 +#define _tmp_158_type 1375 +#define _tmp_159_type 1376 +#define _tmp_160_type 1377 +#define _loop0_162_type 1378 +#define _gather_161_type 1379 +#define _loop0_164_type 1380 +#define _gather_163_type 1381 +#define _loop0_166_type 1382 +#define _gather_165_type 1383 +#define _loop0_168_type 1384 +#define _gather_167_type 1385 +#define _tmp_169_type 1386 +#define _tmp_170_type 1387 +#define _tmp_171_type 1388 +#define _tmp_172_type 1389 +#define _tmp_173_type 1390 +#define _loop0_175_type 1391 +#define _gather_174_type 1392 +#define _tmp_176_type 1393 +#define _tmp_177_type 1394 +#define _tmp_178_type 1395 +#define _tmp_179_type 1396 +#define _tmp_180_type 1397 +#define _tmp_181_type 1398 +#define _tmp_182_type 1399 +#define _tmp_183_type 1400 +#define _tmp_184_type 1401 +#define _tmp_185_type 1402 +#define _tmp_186_type 1403 +#define _tmp_187_type 1404 +#define _tmp_188_type 1405 +#define _tmp_189_type 1406 +#define _tmp_190_type 1407 +#define _tmp_191_type 1408 +#define _tmp_192_type 1409 +#define _tmp_193_type 1410 +#define _tmp_194_type 1411 +#define _tmp_195_type 1412 +#define _tmp_196_type 1413 +#define _tmp_197_type 1414 +#define _tmp_198_type 1415 +#define _tmp_199_type 1416 +#define _tmp_200_type 1417 +#define _tmp_201_type 1418 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -541,6 +542,7 @@ static pattern_ty literal_pattern_rule(Parser *p); static expr_ty literal_expr_rule(Parser *p); static expr_ty complex_number_rule(Parser *p); static expr_ty signed_number_rule(Parser *p); +static expr_ty imaginary_number_rule(Parser *p); static pattern_ty capture_pattern_rule(Parser *p); static expr_ty pattern_capture_target_rule(Parser *p); static pattern_ty wildcard_pattern_rule(Parser *p); @@ -6316,7 +6318,7 @@ literal_expr_rule(Parser *p) return _res; } -// complex_number: signed_number '+' NUMBER | signed_number '-' NUMBER +// complex_number: signed_number '+' imaginary_number | signed_number '-' imaginary_number static expr_ty complex_number_rule(Parser *p) { @@ -6336,12 +6338,12 @@ complex_number_rule(Parser *p) UNUSED(_start_lineno); // Only used by EXTRA macro int _start_col_offset = p->tokens[_mark]->col_offset; UNUSED(_start_col_offset); // Only used by EXTRA macro - { // signed_number '+' NUMBER + { // signed_number '+' imaginary_number if (p->error_indicator) { D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> complex_number[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "signed_number '+' NUMBER")); + D(fprintf(stderr, "%*c> complex_number[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "signed_number '+' imaginary_number")); Token * _literal; expr_ty imag; expr_ty real; @@ -6350,10 +6352,10 @@ complex_number_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 14)) // token='+' && - (imag = _PyPegen_number_token(p)) // NUMBER + (imag = imaginary_number_rule(p)) // imaginary_number ) { - D(fprintf(stderr, "%*c+ complex_number[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "signed_number '+' NUMBER")); + D(fprintf(stderr, "%*c+ complex_number[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "signed_number '+' imaginary_number")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { D(p->level--); @@ -6373,14 +6375,14 @@ complex_number_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s complex_number[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "signed_number '+' NUMBER")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "signed_number '+' imaginary_number")); } - { // signed_number '-' NUMBER + { // signed_number '-' imaginary_number if (p->error_indicator) { D(p->level--); return NULL; } - D(fprintf(stderr, "%*c> complex_number[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "signed_number '-' NUMBER")); + D(fprintf(stderr, "%*c> complex_number[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "signed_number '-' imaginary_number")); Token * _literal; expr_ty imag; expr_ty real; @@ -6389,10 +6391,10 @@ complex_number_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 15)) // token='-' && - (imag = _PyPegen_number_token(p)) // NUMBER + (imag = imaginary_number_rule(p)) // imaginary_number ) { - D(fprintf(stderr, "%*c+ complex_number[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "signed_number '-' NUMBER")); + D(fprintf(stderr, "%*c+ complex_number[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "signed_number '-' imaginary_number")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { D(p->level--); @@ -6412,7 +6414,7 @@ complex_number_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s complex_number[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "signed_number '-' NUMBER")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "signed_number '-' imaginary_number")); } _res = NULL; done: @@ -6501,6 +6503,47 @@ signed_number_rule(Parser *p) return _res; } +// imaginary_number: NUMBER +static expr_ty +imaginary_number_rule(Parser *p) +{ + D(p->level++); + if (p->error_indicator) { + D(p->level--); + return NULL; + } + expr_ty _res = NULL; + int _mark = p->mark; + { // NUMBER + if (p->error_indicator) { + D(p->level--); + return NULL; + } + D(fprintf(stderr, "%*c> imaginary_number[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NUMBER")); + expr_ty imag; + if ( + (imag = _PyPegen_number_token(p)) // NUMBER + ) + { + D(fprintf(stderr, "%*c+ imaginary_number[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NUMBER")); + _res = _PyAST_EnsureImaginary ( imag ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + D(p->level--); + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s imaginary_number[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NUMBER")); + } + _res = NULL; + done: + D(p->level--); + return _res; +} + // capture_pattern: pattern_capture_target static pattern_ty capture_pattern_rule(Parser *p) diff --git a/Parser/pegen.c b/Parser/pegen.c index 398c8480d5ed9e..4d915a95d9be9c 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -1,5 +1,5 @@ #include -#include "pycore_ast.h" // _PyAST_Validate() +#include "pycore_ast.h" // _PyAST_Validate(), _PyAST_EnsureImaginary #include #include "tokenizer.h" diff --git a/Python/ast.c b/Python/ast.c index 57d92a3824a2db..1e68dc960f9174 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -408,7 +408,7 @@ ensure_literal_complex(expr_ty exp) if (exp->v.BinOp.op != Add && exp->v.BinOp.op != Sub) { return 0; } - // Check LHS is a real number + // Check LHS is a real number (potentially signed) switch (left->kind) { case Constant_kind: @@ -424,7 +424,7 @@ ensure_literal_complex(expr_ty exp) default: return 0; } - // Check RHS is an imaginary number + // Check RHS is an imaginary number (no separate sign allowed) switch (right->kind) { case Constant_kind: @@ -432,11 +432,6 @@ ensure_literal_complex(expr_ty exp) return 0; } break; - case UnaryOp_kind: - if (!ensure_literal_negative(right, /*real=*/false, /*imaginary=*/true)) { - return 0; - } - break; default: return 0; } @@ -876,6 +871,16 @@ _PyAST_Validate(mod_ty mod) return res; } +expr_ty +_PyAST_EnsureImaginary(expr_ty exp) +{ + if (exp->kind != Constant_kind || !PyComplex_CheckExact(exp->v.Constant.value)) { + PyErr_SetString(PyExc_SyntaxError, "Imaginary number required in complex literal"); + return NULL; + } + return exp; +} + PyObject * _PyAST_GetDocString(asdl_stmt_seq *body) { From 8ed9847dd552946b758d0ef52c9fe08a008ac9fe Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Wed, 28 Apr 2021 22:16:52 +1000 Subject: [PATCH 36/38] Don't add renamed test case back --- Lib/test/test_patma.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py index 4371bcc3337ffb..f3275529f22dfa 100644 --- a/Lib/test/test_patma.py +++ b/Lib/test/test_patma.py @@ -2429,15 +2429,6 @@ def test_patma_240(self): pass """) - @no_perf - def test_patma_240b(self): - # Ensure mapping keys are also restricted - self.assert_syntax_error(""" - match ...: - case {0+0:_}: - pass - """) - @no_perf def test_patma_241(self): self.assert_syntax_error(""" From 46e7895e4ef88fa15bf58733f4f178f148ca994b Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Wed, 28 Apr 2021 22:19:19 +1000 Subject: [PATCH 37/38] Restore optimisations as compiler is not longer going validation --- Python/ast_opt.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 037b147317b5be..254dd646c467ba 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -805,8 +805,7 @@ astfold_pattern(pattern_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) } switch (node_->kind) { case MatchValue_kind: - // Don't fold the value as an expression, since we need values like - // 0+0 to raise SyntaxErrors in the compiler! + CALL(astfold_expr, expr_ty, node_->v.MatchValue.value); break; case MatchSingleton_kind: break; @@ -814,12 +813,11 @@ astfold_pattern(pattern_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) CALL_SEQ(astfold_pattern, pattern, node_->v.MatchSequence.patterns); break; case MatchMapping_kind: - // Don't fold the keys as expressions, since we need keys like 0+0 - // to raise SyntaxErrors in the compiler! + CALL_SEQ(astfold_expr, expr, node_->v.MatchMapping.keys); CALL_SEQ(astfold_pattern, pattern, node_->v.MatchMapping.patterns); break; case MatchClass_kind: - // No need to fold the class (it's just a dotted name). + CALL(astfold_expr, expr_ty, node_->v.MatchClass.cls); CALL_SEQ(astfold_pattern, pattern, node_->v.MatchClass.patterns); CALL_SEQ(astfold_pattern, pattern, node_->v.MatchClass.kwd_patterns); break; @@ -827,7 +825,7 @@ astfold_pattern(pattern_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) break; case MatchAs_kind: if (node_->v.MatchAs.pattern) { - CALL(astfold_pattern, pattern, node_->v.MatchAs.pattern); + CALL(astfold_pattern, expr_ty, node_->v.MatchAs.pattern); } break; case MatchOr_kind: From 8ba335e08481758cb61ca6ba20dd3b4e402c5f1e Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Wed, 28 Apr 2021 22:42:00 +1000 Subject: [PATCH 38/38] Move syntax checking helper to pegen --- Grammar/python.gram | 2 +- Include/internal/pycore_ast.h | 3 --- Parser/parser.c | 2 +- Parser/pegen.c | 12 +++++++++++- Parser/pegen.h | 1 + Python/ast.c | 10 ---------- 6 files changed, 14 insertions(+), 16 deletions(-) diff --git a/Grammar/python.gram b/Grammar/python.gram index 5fc8f3e999996d..c8d765b632909c 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -284,7 +284,7 @@ signed_number[expr_ty]: | '-' number=NUMBER { _PyAST_UnaryOp(USub, number, EXTRA) } imaginary_number[expr_ty]: - | imag=NUMBER { _PyAST_EnsureImaginary(imag) } + | imag=NUMBER { _PyPegen_ensure_imaginary(p, imag) } capture_pattern[pattern_ty]: | target=pattern_capture_target { _PyAST_MatchAs(NULL, target->v.Name.id, EXTRA) } diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index 05fca96c75f61f..ebb6a90087bb52 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -841,9 +841,6 @@ int PyAST_Check(PyObject* obj); extern int _PyAST_Validate(mod_ty); -/* Helper to let the parser reject 0+0 (et al) when it wants a complex literal */ -extern expr_ty _PyAST_EnsureImaginary(expr_ty); - /* _PyAST_ExprAsUnicode is defined in ast_unparse.c */ extern PyObject* _PyAST_ExprAsUnicode(expr_ty); diff --git a/Parser/parser.c b/Parser/parser.c index c52da7d2f262c2..5baf82bc627d11 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -6526,7 +6526,7 @@ imaginary_number_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ imaginary_number[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NUMBER")); - _res = _PyAST_EnsureImaginary ( imag ); + _res = _PyPegen_ensure_imaginary ( p , imag ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; D(p->level--); diff --git a/Parser/pegen.c b/Parser/pegen.c index 4d915a95d9be9c..4d6e69edc23304 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -1,5 +1,5 @@ #include -#include "pycore_ast.h" // _PyAST_Validate(), _PyAST_EnsureImaginary +#include "pycore_ast.h" // _PyAST_Validate(), #include #include "tokenizer.h" @@ -2348,6 +2348,16 @@ _PyPegen_concatenate_strings(Parser *p, asdl_seq *strings) return NULL; } +expr_ty +_PyPegen_ensure_imaginary(Parser *p, expr_ty exp) +{ + if (exp->kind != Constant_kind || !PyComplex_CheckExact(exp->v.Constant.value)) { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(exp, "Imaginary number required in complex literal"); + return NULL; + } + return exp; +} + mod_ty _PyPegen_make_module(Parser *p, asdl_stmt_seq *a) { asdl_type_ignore_seq *type_ignores = NULL; diff --git a/Parser/pegen.h b/Parser/pegen.h index ccb8099f319d79..2af551b706993f 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -283,6 +283,7 @@ expr_ty _PyPegen_collect_call_seqs(Parser *, asdl_expr_seq *, asdl_seq *, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); expr_ty _PyPegen_concatenate_strings(Parser *p, asdl_seq *); +expr_ty _PyPegen_ensure_imaginary(Parser *p, expr_ty); asdl_seq *_PyPegen_join_sequences(Parser *, asdl_seq *, asdl_seq *); int _PyPegen_check_barry_as_flufl(Parser *, Token *); mod_ty _PyPegen_make_module(Parser *, asdl_stmt_seq *); diff --git a/Python/ast.c b/Python/ast.c index 1e68dc960f9174..1fc83f6301962d 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -871,16 +871,6 @@ _PyAST_Validate(mod_ty mod) return res; } -expr_ty -_PyAST_EnsureImaginary(expr_ty exp) -{ - if (exp->kind != Constant_kind || !PyComplex_CheckExact(exp->v.Constant.value)) { - PyErr_SetString(PyExc_SyntaxError, "Imaginary number required in complex literal"); - return NULL; - } - return exp; -} - PyObject * _PyAST_GetDocString(asdl_stmt_seq *body) {