Fixed bug #76437 (token_get_all with TOKEN_PARSE flag fails to recognise close tag)

Author: laruence

NEWS

@@ -27,6 +27,8 @@ PHP                                                                        NEWS
     `!(zval_gc_flags((str)->gc)). (Nikita, Laruence)
 
 - Tokenizer:
+  . Fixed bug #76437 (token_get_all with TOKEN_PARSE flag fails to recognise
+    close tag). (Laruence)
   . Fixed bug #75218 (Change remaining uncatchable fatal errors for parsing
     into ParseError). (Nikita)
 

ext/tokenizer/tests/bug76437.phpt

@@ -0,0 +1,34 @@
+--TEST--
+Bug #76437 (token_get_all with TOKEN_PARSE flag fails to recognise close tag)
+--SKIPIF--
+<?php if (!extension_loaded("tokenizer")) print "skip"; ?>
+--FILE--
+<?php
+$open_tag1 = token_get_all('<?=$a?>')[0];
+$open_tag2 = token_get_all('<?=$a?>', TOKEN_PARSE)[0];
+var_dump($open_tag1);
+var_dump($open_tag1 === $open_tag2);
+$open_tag1 = token_get_all('<?php echo 2; ?>')[6];
+$open_tag2 = token_get_all('<?php echo 2; ?>', TOKEN_PARSE)[6];
+var_dump($open_tag1);
+var_dump($open_tag1 === $open_tag2);
+?>
+--EXPECT--
+array(3) {
+  [0]=>
+  int(380)
+  [1]=>
+  string(3) "<?="
+  [2]=>
+  int(1)
+}
+bool(true)
+array(3) {
+  [0]=>
+  int(381)
+  [1]=>
+  string(2) "?>"
+  [2]=>
+  int(1)
+}
+bool(true)

ext/tokenizer/tokenizer.c

@@ -191,8 +191,16 @@ void on_event(zend_php_scanner_event event, int token, int line, void *context)
 
 	switch (event) {
 		case ON_TOKEN:
-			if (token == END) break;
-			add_token(token_stream, token, LANG_SCNG(yy_text), LANG_SCNG(yy_leng), line);
+			{
+				if (token == END) break;
+				/* Specical cases */
+				if (token == ';' && LANG_SCNG(yy_leng) == sizeof("?>") - 1) {
+					token = T_CLOSE_TAG;
+				} else if (token == T_ECHO && LANG_SCNG(yy_leng) == sizeof("<?=") - 1) {
+					token = T_OPEN_TAG_WITH_ECHO;
+				}
+				add_token(token_stream, token, LANG_SCNG(yy_text), LANG_SCNG(yy_leng), line);
+			}
 			break;
 		case ON_FEEDBACK:
 			tokens_ht = Z_ARRVAL_P(token_stream);