Generate tokenizer data and add test

Author: nikic

ext/tokenizer/tests/namespaced_names.phpt

@@ -0,0 +1,39 @@
+--TEST--
+Tokenization of namespaced names
+--FILE--
+<?php
+
+$code = <<<'CODE'
+<?php
+Foo
+Foo\Bar
+\Foo\Bar
+namespace\Foo
+Foo \ Bar
+CODE;
+
+foreach (PhpToken::getAll($code) as $token) {
+    echo "{$token->getTokenName()}: \"$token->text\"\n";
+}
+
+?>
+--EXPECT--
+T_OPEN_TAG: "<?php
+"
+T_STRING: "Foo"
+T_WHITESPACE: "
+"
+T_NAME_QUALIFIED: "Foo\Bar"
+T_WHITESPACE: "
+"
+T_NAME_FULLY_QUALIFIED: "\Foo\Bar"
+T_WHITESPACE: "
+"
+T_NAME_RELATIVE: "namespace\Foo"
+T_WHITESPACE: "
+"
+T_STRING: "Foo"
+T_WHITESPACE: " "
+T_NS_SEPARATOR: "\"
+T_WHITESPACE: " "
+T_STRING: "Bar"

ext/tokenizer/tokenizer_data.c

@@ -77,6 +77,9 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
 	REGISTER_LONG_CONSTANT("T_LNUMBER", T_LNUMBER, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_DNUMBER", T_DNUMBER, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_STRING", T_STRING, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_NAME_FULLY_QUALIFIED", T_NAME_FULLY_QUALIFIED, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_NAME_RELATIVE", T_NAME_RELATIVE, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_NAME_QUALIFIED", T_NAME_QUALIFIED, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_VARIABLE", T_VARIABLE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_INLINE_HTML", T_INLINE_HTML, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_ENCAPSED_AND_WHITESPACE", T_ENCAPSED_AND_WHITESPACE, CONST_CS | CONST_PERSISTENT);
@@ -221,6 +224,9 @@ char *get_token_type_name(int token_type)
 		case T_LNUMBER: return "T_LNUMBER";
 		case T_DNUMBER: return "T_DNUMBER";
 		case T_STRING: return "T_STRING";
+		case T_NAME_FULLY_QUALIFIED: return "T_NAME_FULLY_QUALIFIED";
+		case T_NAME_RELATIVE: return "T_NAME_RELATIVE";
+		case T_NAME_QUALIFIED: return "T_NAME_QUALIFIED";
 		case T_VARIABLE: return "T_VARIABLE";
 		case T_INLINE_HTML: return "T_INLINE_HTML";
 		case T_ENCAPSED_AND_WHITESPACE: return "T_ENCAPSED_AND_WHITESPACE";