diff --git a/Zend/tests/gh14926.phpt b/Zend/tests/gh14926.phpt new file mode 100644 index 0000000000000..8c610bbf4b86f --- /dev/null +++ b/Zend/tests/gh14926.phpt @@ -0,0 +1,22 @@ +--TEST-- +yield from is tokenized separately +--FILE-- + +--EXPECT-- +yield| |from| |$foo|;| +yield| |# comment| +|/* other comment */| |from| |$foo|;| diff --git a/Zend/zend_language_parser.y b/Zend/zend_language_parser.y index f7822caa88dc3..e77758ba62a4f 100644 --- a/Zend/zend_language_parser.y +++ b/Zend/zend_language_parser.y @@ -60,7 +60,7 @@ static YYSIZE_T zend_yytnamerr(char*, const char*); %precedence T_PRINT %precedence T_YIELD %precedence T_DOUBLE_ARROW -%precedence T_YIELD_FROM +%precedence T_FROM %precedence '=' T_PLUS_EQUAL T_MINUS_EQUAL T_MUL_EQUAL T_DIV_EQUAL T_CONCAT_EQUAL T_MOD_EQUAL T_AND_EQUAL T_OR_EQUAL T_XOR_EQUAL T_SL_EQUAL T_SR_EQUAL T_POW_EQUAL T_COALESCE_EQUAL %left '?' ':' %right T_COALESCE @@ -109,7 +109,7 @@ static YYSIZE_T zend_yytnamerr(char*, const char*); %token T_LOGICAL_AND "'and'" %token T_PRINT "'print'" %token T_YIELD "'yield'" -%token T_YIELD_FROM "'yield from'" +%token T_FROM "'from'" %token T_INSTANCEOF "'instanceof'" %token T_NEW "'new'" %token T_CLONE "'clone'" @@ -1312,7 +1312,7 @@ expr: | T_YIELD { $$ = zend_ast_create(ZEND_AST_YIELD, NULL, NULL); CG(extra_fn_flags) |= ZEND_ACC_GENERATOR; } | T_YIELD expr { $$ = zend_ast_create(ZEND_AST_YIELD, $2, NULL); CG(extra_fn_flags) |= ZEND_ACC_GENERATOR; } | T_YIELD expr T_DOUBLE_ARROW expr { $$ = zend_ast_create(ZEND_AST_YIELD, $4, $2); CG(extra_fn_flags) |= ZEND_ACC_GENERATOR; } - | T_YIELD_FROM expr { $$ = zend_ast_create(ZEND_AST_YIELD_FROM, $2); CG(extra_fn_flags) |= ZEND_ACC_GENERATOR; } + | T_YIELD T_FROM expr { $$ = zend_ast_create(ZEND_AST_YIELD_FROM, $3); CG(extra_fn_flags) |= ZEND_ACC_GENERATOR; } | T_THROW expr { $$ = zend_ast_create(ZEND_AST_THROW, $2); } | inline_function { $$ = $1; } | attributes inline_function { $$ = zend_ast_with_attributes($2, $1); } diff --git a/Zend/zend_language_scanner.l b/Zend/zend_language_scanner.l index 4551d26a17e79..f7de8b4b37f98 100644 --- a/Zend/zend_language_scanner.l +++ b/Zend/zend_language_scanner.l @@ -1412,9 +1412,14 @@ OPTIONAL_WHITESPACE_OR_COMMENTS ({WHITESPACE}|{MULTI_LINE_COMMENT}|{SINGLE_LINE_ } "yield"{WHITESPACE_OR_COMMENTS}"from"[^a-zA-Z0-9_\x80-\xff] { - yyless(yyleng - 1); - HANDLE_NEWLINES(yytext, yyleng); - RETURN_TOKEN_WITH_IDENT(T_YIELD_FROM); + yyless(5); + yy_push_state(ST_LOOKING_FOR_FROM); + RETURN_TOKEN_WITH_IDENT(T_YIELD); +} + +"from" { + yy_pop_state(); + RETURN_TOKEN_WITH_IDENT(T_FROM); } "yield" { @@ -1580,7 +1585,7 @@ OPTIONAL_WHITESPACE_OR_COMMENTS ({WHITESPACE}|{MULTI_LINE_COMMENT}|{SINGLE_LINE_ RETURN_TOKEN(T_NULLSAFE_OBJECT_OPERATOR); } -{WHITESPACE}+ { +{WHITESPACE}+ { goto return_whitespace; } @@ -2383,7 +2388,7 @@ inline_char_handler: } -"#"|"//" { +"#"|"//" { while (YYCURSOR < YYLIMIT) { switch (*YYCURSOR++) { case '\r': @@ -2407,7 +2412,7 @@ inline_char_handler: RETURN_OR_SKIP_TOKEN(T_COMMENT); } -"/*"|"/**"{WHITESPACE} { +"/*"|"/**"{WHITESPACE} { int doc_com; if (yyleng > 2) { @@ -2443,7 +2448,7 @@ inline_char_handler: RETURN_OR_SKIP_TOKEN(T_COMMENT); } -{ANY_CHAR} { +{ANY_CHAR} { yyless(0); yy_pop_state(); goto restart; diff --git a/ext/tokenizer/tokenizer_data.c b/ext/tokenizer/tokenizer_data.c index cdaaddecd7bfa..105c977eda743 100644 --- a/ext/tokenizer/tokenizer_data.c +++ b/ext/tokenizer/tokenizer_data.c @@ -47,7 +47,7 @@ char *get_token_type_name(int token_type) case T_LOGICAL_AND: return "T_LOGICAL_AND"; case T_PRINT: return "T_PRINT"; case T_YIELD: return "T_YIELD"; - case T_YIELD_FROM: return "T_YIELD_FROM"; + case T_FROM: return "T_FROM"; case T_INSTANCEOF: return "T_INSTANCEOF"; case T_NEW: return "T_NEW"; case T_CLONE: return "T_CLONE"; diff --git a/ext/tokenizer/tokenizer_data.stub.php b/ext/tokenizer/tokenizer_data.stub.php index 81e4e92626f37..37c32ef6387bc 100644 --- a/ext/tokenizer/tokenizer_data.stub.php +++ b/ext/tokenizer/tokenizer_data.stub.php @@ -114,9 +114,9 @@ const T_YIELD = UNKNOWN; /** * @var int - * @cvalue T_YIELD_FROM + * @cvalue T_FROM */ -const T_YIELD_FROM = UNKNOWN; +const T_FROM = UNKNOWN; /** * @var int * @cvalue T_INSTANCEOF diff --git a/ext/tokenizer/tokenizer_data_arginfo.h b/ext/tokenizer/tokenizer_data_arginfo.h index 0fc1f219619dd..e310ab496c0fc 100644 --- a/ext/tokenizer/tokenizer_data_arginfo.h +++ b/ext/tokenizer/tokenizer_data_arginfo.h @@ -1,5 +1,5 @@ /* This is a generated file, edit the .stub.php file instead. - * Stub hash: b7a13b3b242bd535f62a7d819eb0df751efb54ed */ + * Stub hash: 5b33d2752aaf690422935e2d741a099726806297 */ static void register_tokenizer_data_symbols(int module_number) { @@ -25,7 +25,7 @@ static void register_tokenizer_data_symbols(int module_number) REGISTER_LONG_CONSTANT("T_LOGICAL_AND", T_LOGICAL_AND, CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_PRINT", T_PRINT, CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_YIELD", T_YIELD, CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_YIELD_FROM", T_YIELD_FROM, CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_FROM", T_FROM, CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_INSTANCEOF", T_INSTANCEOF, CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_NEW", T_NEW, CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_CLONE", T_CLONE, CONST_PERSISTENT);