Skip to content

Treat namespaced names as single token (reduced version) #5827

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Zend/tests/attributes/019_variable_attribute_name.phpt
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,4 @@ class A {}

?>
--EXPECTF--
Parse error: syntax error, unexpected variable "$x", expecting identifier or "static" or "namespace" or "\" in %s on line %d
Parse error: syntax error, unexpected variable "$x" in %s on line %d
2 changes: 1 addition & 1 deletion Zend/tests/bug43343.phpt
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,4 @@ $foo = 'bar';
var_dump(new namespace::$foo);
?>
--EXPECTF--
Parse error: %s error%sexpecting%s"\"%sin %sbug43343.php on line 5
Parse error: syntax error, unexpected token "namespace" in %s on line %d
2 changes: 1 addition & 1 deletion Zend/tests/bug55086.phpt
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ namespace N2 {
echo $a->hello(), PHP_EOL;
echo $a->foo(), PHP_EOL;
try {
} catch(namespace \Foo $e)
} catch (namespace\Foo $e)
{
}
}
Expand Down
7 changes: 5 additions & 2 deletions Zend/tests/grammar/regression_010.phpt
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,11 @@ Test to check regressions on T_IMPLEMENTS followed by a T_NS_SEPARATOR

interface A{}

// No longer considered legal in PHP 8.
class B implements\A {}

echo "Done", PHP_EOL;
--EXPECT--
Done

?>
--EXPECTF--
Parse error: syntax error, unexpected namespaced name "implements\A", expecting "{" in %s on line %d
10 changes: 10 additions & 0 deletions Zend/tests/namespace_name_namespace.phpt
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
--TEST--
Cannot use "namespace" as namespace name, due to conflict with ns-relative names
--FILE--
<?php

namespace NAMEspace;

?>
--EXPECTF--
Fatal error: Cannot use 'NAMEspace' as namespace name in %s on line %d
10 changes: 10 additions & 0 deletions Zend/tests/namespace_name_namespace_start.phpt
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
--TEST--
Cannot use "namespace\xyz" as namespace name, due to conflict with ns-relative names
--FILE--
<?php

namespace NAMEspace\xyz;

?>
--EXPECTF--
Parse error: syntax error, unexpected namespace-relative name "NAMEspace\xyz", expecting "{" in %s on line %d
37 changes: 37 additions & 0 deletions Zend/tests/namespace_name_reserved_keywords.phpt
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
--TEST--
Reserved keywords in namespace name
--FILE--
<?php

namespace iter\fn {
function test() {
echo __FUNCTION__, "\n";
}
}

namespace fn {
function test() {
echo __FUNCTION__, "\n";
}
}

namespace self {
function test() {
echo __FUNCTION__, "\n";
}
}

namespace {
use iter\fn;
use function fn\test as test2;
use function self\test as test3;
fn\test();
test2();
test3();
}

?>
--EXPECT--
iter\fn\test
fn\test
self\test
10 changes: 10 additions & 0 deletions Zend/tests/namespaced_name_whitespace.phpt
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
--TEST--
Whitespace between namespace separators is no longer allowed
--FILE--
<?php

Foo \ Bar \ Baz;

?>
--EXPECTF--
Parse error: syntax error, unexpected token "\" in %s on line %d
2 changes: 1 addition & 1 deletion Zend/tests/ns_096.phpt
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,4 @@ use Foo\Bar\{\Baz};

?>
--EXPECTF--
Parse error: syntax error, unexpected token "\", expecting identifier or "function" or "const" in %s on line %d
Parse error: syntax error, unexpected fully qualified name "\Baz", expecting identifier or namespaced name or "function" or "const" in %s on line %d
2 changes: 1 addition & 1 deletion Zend/tests/ns_trailing_comma_error_01.phpt
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ Group use declarations mustn't be empty
use Baz\{};
?>
--EXPECTF--
Parse error: syntax error, unexpected token "}", expecting identifier or "function" or "const" in %s on line %d
Parse error: syntax error, unexpected token "}", expecting identifier or namespaced name or "function" or "const" in %s on line %d
2 changes: 1 addition & 1 deletion Zend/tests/ns_trailing_comma_error_02.phpt
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ Group use declarations mustn't contain just a comma
use Baz\{,};
?>
--EXPECTF--
Parse error: syntax error, unexpected token ",", expecting identifier or "function" or "const" in %s on line %d
Parse error: syntax error, unexpected token ",", expecting identifier or namespaced name or "function" or "const" in %s on line %d
2 changes: 1 addition & 1 deletion Zend/tests/ns_trailing_comma_error_04.phpt
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ Group use declarations mustn't begin with a comma
use Baz\{,Foo};
?>
--EXPECTF--
Parse error: syntax error, unexpected token ",", expecting identifier or "function" or "const" in %s on line %d
Parse error: syntax error, unexpected token ",", expecting identifier or namespaced name or "function" or "const" in %s on line %d
2 changes: 1 addition & 1 deletion Zend/tests/ns_trailing_comma_error_07.phpt
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ Unmixed group use declarations mustn't begin with a comma
use function Baz\{,Foo};
?>
--EXPECTF--
Parse error: syntax error, unexpected token ",", expecting identifier in %s on line %d
Parse error: syntax error, unexpected token ",", expecting identifier or namespaced name in %s on line %d
10 changes: 0 additions & 10 deletions Zend/tests/special_name_error1.phpt

This file was deleted.

2 changes: 1 addition & 1 deletion Zend/zend_compile.c
Original file line number Diff line number Diff line change
Expand Up @@ -7432,7 +7432,7 @@ void zend_compile_namespace(zend_ast *ast) /* {{{ */
if (name_ast) {
name = zend_ast_get_str(name_ast);

if (ZEND_FETCH_CLASS_DEFAULT != zend_get_class_fetch_type(name)) {
if (zend_string_equals_literal_ci(name, "namespace")) {
zend_error_noreturn(E_COMPILE_ERROR, "Cannot use '%s' as namespace name", ZSTR_VAL(name));
}

Expand Down
47 changes: 31 additions & 16 deletions Zend/zend_language_parser.y
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,9 @@ static YYSIZE_T zend_yytnamerr(char*, const char*);
%token <ast> T_LNUMBER "integer"
%token <ast> T_DNUMBER "floating-point number"
%token <ast> T_STRING "identifier"
%token <ast> T_NAME_FULLY_QUALIFIED "fully qualified name"
%token <ast> T_NAME_RELATIVE "namespace-relative name"
%token <ast> T_NAME_QUALIFIED "namespaced name"
%token <ast> T_VARIABLE "variable"
%token <ast> T_INLINE_HTML
%token <ast> T_ENCAPSED_AND_WHITESPACE "string content"
Expand Down Expand Up @@ -231,7 +234,7 @@ static YYSIZE_T zend_yytnamerr(char*, const char*);
%token T_ERROR

%type <ast> top_statement namespace_name name statement function_declaration_statement
%type <ast> class_declaration_statement trait_declaration_statement
%type <ast> class_declaration_statement trait_declaration_statement legacy_namespace_name
%type <ast> interface_declaration_statement interface_extends_list
%type <ast> group_use_declaration inline_use_declarations inline_use_declaration
%type <ast> mixed_group_use_declaration use_declaration unprefixed_use_declaration
Expand Down Expand Up @@ -261,7 +264,7 @@ static YYSIZE_T zend_yytnamerr(char*, const char*);
%type <ast> identifier type_expr_without_static union_type_without_static
%type <ast> inline_function union_type
%type <ast> attributed_statement attributed_class_statement attributed_parameter
%type <ast> attribute_decl attribute attributes
%type <ast> attribute_decl attribute attributes namespace_declaration_name
%type <ast> match match_arm_list non_empty_match_arm_list match_arm match_arm_cond_list

%type <num> returns_ref function fn is_reference is_variadic variable_modifiers
Expand Down Expand Up @@ -308,15 +311,29 @@ top_statement_list:
| %empty { $$ = zend_ast_create_list(0, ZEND_AST_STMT_LIST); }
;

/* Name usable in a namespace declaration. */
namespace_declaration_name:
identifier { $$ = $1; }
| T_NAME_QUALIFIED { $$ = $1; }
;

/* Name usable in "use" declarations (loading separator forbidden). */
namespace_name:
T_STRING { $$ = $1; }
| namespace_name T_NS_SEPARATOR T_STRING { $$ = zend_ast_append_str($1, $3); }
| T_NAME_QUALIFIED { $$ = $1; }
;

/* Name usable in "use" declarations (leading separator allowed). */
legacy_namespace_name:
namespace_name { $$ = $1; }
| T_NAME_FULLY_QUALIFIED { $$ = $1; }
;

name:
namespace_name { $$ = $1; $$->attr = ZEND_NAME_NOT_FQ; }
| T_NAMESPACE T_NS_SEPARATOR namespace_name { $$ = $3; $$->attr = ZEND_NAME_RELATIVE; }
| T_NS_SEPARATOR namespace_name { $$ = $2; $$->attr = ZEND_NAME_FQ; }
T_STRING { $$ = $1; $$->attr = ZEND_NAME_NOT_FQ; }
| T_NAME_QUALIFIED { $$ = $1; $$->attr = ZEND_NAME_NOT_FQ; }
Copy link
Contributor

@TysonAndre TysonAndre Jul 20, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One minor oversight in the implementation that should be easy to address (It's easy to forget about. I assume the author will address this, and supporting tokens would generally be assumed to be part of the final implementation of any RFC adding a new token type, for anyone wondering. I've made this mistake in drafts of other unrelated RFC implementations I've worked on):

  1. Constants should be provided to userland for T_NAME_QUALIFIED, T_NAME_FULLY_QUALIFIED, and T_NAME_RELATIVE
  2. The implementation of char *get_token_type_name should be updated with the new token types, so that calls from PHP to token_name() (and possibly error messages)

At least 2 parts of ext/tokenizer/tokenizer_data.c need to be updated

void tokenizer_register_constants(INIT_FUNC_ARGS) {
	REGISTER_LONG_CONSTANT("T_THROW", T_THROW, CONST_CS | CONST_PERSISTENT);
	REGISTER_LONG_CONSTANT("T_INCLUDE", T_INCLUDE, CONST_CS | CONST_PERSISTENT);
	REGISTER_LONG_CONSTANT("T_INCLUDE_ONCE", T_INCLUDE_ONCE, CONST_CS | CONST_PERSISTENT);
char *get_token_type_name(int token_type)
{
	switch (token_type) {

		case T_THROW: return "T_THROW";
		case T_INCLUDE: return "T_INCLUDE";
		case T_INCLUDE_ONCE: return "T_INCLUDE_ONCE";

Currently (the exact value may be different, I also merged another RFC implementation for this php build),

php > echo token_name(T_NAME_QUALIFIED);

Warning: Uncaught Error: Undefined constant "T_NAME_QUALIFIED" in php shell code:1
Stack trace:
#0 {main}
  thrown in php shell code on line 1
php > echo json_encode(token_get_all('<?php use Foo\Bar as X;'));
[[389,"<?php ",1],[352,"use",1],[392," ",1],[314,"Foo\\Bar",1],[392," ",1],[336,"as",1],[392," ",1],[311,"X",1],";"]
php > echo token_name(314);
UNKNOWN

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, I had indeed forgotten about this. Should be fixed now!

| T_NAME_FULLY_QUALIFIED { $$ = $1; $$->attr = ZEND_NAME_FQ; }
| T_NAME_RELATIVE { $$ = $1; $$->attr = ZEND_NAME_RELATIVE; }
;

attribute_decl:
Expand Down Expand Up @@ -350,10 +367,10 @@ top_statement:
{ $$ = zend_ast_create(ZEND_AST_HALT_COMPILER,
zend_ast_create_zval_from_long(zend_get_scanned_file_offset()));
zend_stop_lexing(); }
| T_NAMESPACE namespace_name ';'
| T_NAMESPACE namespace_declaration_name ';'
{ $$ = zend_ast_create(ZEND_AST_NAMESPACE, $2, NULL);
RESET_DOC_COMMENT(); }
| T_NAMESPACE namespace_name { RESET_DOC_COMMENT(); }
| T_NAMESPACE namespace_declaration_name { RESET_DOC_COMMENT(); }
'{' top_statement_list '}'
{ $$ = zend_ast_create(ZEND_AST_NAMESPACE, $2, $5); }
| T_NAMESPACE { RESET_DOC_COMMENT(); }
Expand All @@ -372,17 +389,13 @@ use_type:
;

group_use_declaration:
namespace_name T_NS_SEPARATOR '{' unprefixed_use_declarations possible_comma '}'
legacy_namespace_name T_NS_SEPARATOR '{' unprefixed_use_declarations possible_comma '}'
{ $$ = zend_ast_create(ZEND_AST_GROUP_USE, $1, $4); }
| T_NS_SEPARATOR namespace_name T_NS_SEPARATOR '{' unprefixed_use_declarations possible_comma '}'
{ $$ = zend_ast_create(ZEND_AST_GROUP_USE, $2, $5); }
;

mixed_group_use_declaration:
namespace_name T_NS_SEPARATOR '{' inline_use_declarations possible_comma '}'
legacy_namespace_name T_NS_SEPARATOR '{' inline_use_declarations possible_comma '}'
{ $$ = zend_ast_create(ZEND_AST_GROUP_USE, $1, $4);}
| T_NS_SEPARATOR namespace_name T_NS_SEPARATOR '{' inline_use_declarations possible_comma '}'
{ $$ = zend_ast_create(ZEND_AST_GROUP_USE, $2, $5); }
;

possible_comma:
Expand Down Expand Up @@ -424,8 +437,10 @@ unprefixed_use_declaration:
;

use_declaration:
unprefixed_use_declaration { $$ = $1; }
| T_NS_SEPARATOR unprefixed_use_declaration { $$ = $2; }
legacy_namespace_name
{ $$ = zend_ast_create(ZEND_AST_USE_ELEM, $1, NULL); }
| legacy_namespace_name T_AS T_STRING
{ $$ = zend_ast_create(ZEND_AST_USE_ELEM, $1, $3); }
;

const_list:
Expand Down
20 changes: 16 additions & 4 deletions Zend/zend_language_scanner.l
Original file line number Diff line number Diff line change
Expand Up @@ -1596,10 +1596,6 @@ NEWLINE ("\r"|"\n"|"\r\n")
RETURN_TOKEN(T_PAAMAYIM_NEKUDOTAYIM);
}

<ST_IN_SCRIPTING>"\\" {
RETURN_TOKEN(T_NS_SEPARATOR);
}

<ST_IN_SCRIPTING>"..." {
RETURN_TOKEN(T_ELLIPSIS);
}
Expand Down Expand Up @@ -2288,6 +2284,22 @@ inline_char_handler:
RETURN_TOKEN_WITH_VAL(T_ENCAPSED_AND_WHITESPACE);
}

<ST_IN_SCRIPTING>"namespace"("\\"{LABEL})+ {
RETURN_TOKEN_WITH_STR(T_NAME_RELATIVE, sizeof("namespace\\") - 1);
}

<ST_IN_SCRIPTING>{LABEL}("\\"{LABEL})+ {
RETURN_TOKEN_WITH_STR(T_NAME_QUALIFIED, 0);
}

<ST_IN_SCRIPTING>"\\"{LABEL}("\\"{LABEL})* {
RETURN_TOKEN_WITH_STR(T_NAME_FULLY_QUALIFIED, 1);
}

<ST_IN_SCRIPTING>"\\" {
RETURN_TOKEN(T_NS_SEPARATOR);
}

<ST_IN_SCRIPTING,ST_VAR_OFFSET>{LABEL} {
RETURN_TOKEN_WITH_STR(T_STRING, 0);
}
Expand Down
39 changes: 39 additions & 0 deletions ext/tokenizer/tests/namespaced_names.phpt
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
--TEST--
Tokenization of namespaced names
--FILE--
<?php

$code = <<<'CODE'
<?php
Foo
Foo\Bar
\Foo\Bar
namespace\Foo
Foo \ Bar
CODE;

foreach (PhpToken::getAll($code) as $token) {
echo "{$token->getTokenName()}: \"$token->text\"\n";
}

?>
--EXPECT--
T_OPEN_TAG: "<?php
"
T_STRING: "Foo"
T_WHITESPACE: "
"
T_NAME_QUALIFIED: "Foo\Bar"
T_WHITESPACE: "
"
T_NAME_FULLY_QUALIFIED: "\Foo\Bar"
T_WHITESPACE: "
"
T_NAME_RELATIVE: "namespace\Foo"
T_WHITESPACE: "
"
T_STRING: "Foo"
T_WHITESPACE: " "
T_NS_SEPARATOR: "\"
T_WHITESPACE: " "
T_STRING: "Bar"
6 changes: 6 additions & 0 deletions ext/tokenizer/tokenizer_data.c
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,9 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
REGISTER_LONG_CONSTANT("T_LNUMBER", T_LNUMBER, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_DNUMBER", T_DNUMBER, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_STRING", T_STRING, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_NAME_FULLY_QUALIFIED", T_NAME_FULLY_QUALIFIED, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_NAME_RELATIVE", T_NAME_RELATIVE, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_NAME_QUALIFIED", T_NAME_QUALIFIED, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_VARIABLE", T_VARIABLE, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_INLINE_HTML", T_INLINE_HTML, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_ENCAPSED_AND_WHITESPACE", T_ENCAPSED_AND_WHITESPACE, CONST_CS | CONST_PERSISTENT);
Expand Down Expand Up @@ -221,6 +224,9 @@ char *get_token_type_name(int token_type)
case T_LNUMBER: return "T_LNUMBER";
case T_DNUMBER: return "T_DNUMBER";
case T_STRING: return "T_STRING";
case T_NAME_FULLY_QUALIFIED: return "T_NAME_FULLY_QUALIFIED";
case T_NAME_RELATIVE: return "T_NAME_RELATIVE";
case T_NAME_QUALIFIED: return "T_NAME_QUALIFIED";
case T_VARIABLE: return "T_VARIABLE";
case T_INLINE_HTML: return "T_INLINE_HTML";
case T_ENCAPSED_AND_WHITESPACE: return "T_ENCAPSED_AND_WHITESPACE";
Expand Down