PHP 8.0: handle changed tokenization of namespaced names

> Namespaced names are now represented using the `T_NAME_QUALIFIED` (`Foo\Bar`), `T_NAME_FULLY_QUALIFIED` (`\Foo\Bar`) and `T_NAME_RELATIVE` (`namespace\Foo\Bar`) tokens. `T_NS_SEPARATOR` is only used for standalone namespace separators, and only syntactially valid in conjunction with group use declarations.

Ref: https://www.php.net/manual/en/migration80.incompatible.php#migration80.incompatible.tokenizer

This commit:
* Adds recognition for the new PHP 8.0 tokens for namespaced names and highlights these as `TOKEN_DEFAULT`, same as a non-namespaced `T_STRING` name.
* Adjusts the new unit tests to document the different highlighting between PHP < 8.0 and PHP 8.0+.

I've chosen not to make force the same highlighting across versions, but to let the highlighting respect the different tokenization across PHP versions.

Author: jrfnl

src/Highlighter.php

@@ -118,46 +118,8 @@ private function tokenize($source)
 
         foreach ($tokens as $token) {
             if (is_array($token)) {
-                switch ($token[0]) {
-                    case T_WHITESPACE:
-                        break;
-
-                    case T_OPEN_TAG:
-                    case T_OPEN_TAG_WITH_ECHO:
-                    case T_CLOSE_TAG:
-                    case T_STRING:
-                    case T_VARIABLE:
-
-                    // Constants
-                    case T_DIR:
-                    case T_FILE:
-                    case T_METHOD_C:
-                    case T_DNUMBER:
-                    case T_LNUMBER:
-                    case T_NS_C:
-                    case T_LINE:
-                    case T_CLASS_C:
-                    case T_FUNC_C:
-                    case T_TRAIT_C:
-                        $newType = self::TOKEN_DEFAULT;
-                        break;
-
-                    case T_COMMENT:
-                    case T_DOC_COMMENT:
-                        $newType = self::TOKEN_COMMENT;
-                        break;
-
-                    case T_ENCAPSED_AND_WHITESPACE:
-                    case T_CONSTANT_ENCAPSED_STRING:
-                        $newType = self::TOKEN_STRING;
-                        break;
-
-                    case T_INLINE_HTML:
-                        $newType = self::TOKEN_HTML;
-                        break;
-
-                    default:
-                        $newType = self::TOKEN_KEYWORD;
+                if ($token[0] !== T_WHITESPACE) {
+                    $newType = $this->getTokenType($token);
                 }
             } else {
                 $newType = $token === '"' ? self::TOKEN_STRING : self::TOKEN_KEYWORD;
@@ -183,6 +145,59 @@ private function tokenize($source)
         return $output;
     }
 
+    /**
+     * @param array $arrayToken
+     * @return string
+     */
+    private function getTokenType($arrayToken)
+    {
+        switch ($arrayToken[0]) {
+            case T_OPEN_TAG:
+            case T_OPEN_TAG_WITH_ECHO:
+            case T_CLOSE_TAG:
+            case T_STRING:
+            case T_VARIABLE:
+
+            // Constants
+            case T_DIR:
+            case T_FILE:
+            case T_METHOD_C:
+            case T_DNUMBER:
+            case T_LNUMBER:
+            case T_NS_C:
+            case T_LINE:
+            case T_CLASS_C:
+            case T_FUNC_C:
+            case T_TRAIT_C:
+                return self::TOKEN_DEFAULT;
+
+            case T_COMMENT:
+            case T_DOC_COMMENT:
+                return self::TOKEN_COMMENT;
+
+            case T_ENCAPSED_AND_WHITESPACE:
+            case T_CONSTANT_ENCAPSED_STRING:
+                return self::TOKEN_STRING;
+
+            case T_INLINE_HTML:
+                return self::TOKEN_HTML;
+        }
+
+        // Handle PHP >= 8.0 namespaced name tokens.
+        // https://www.php.net/manual/en/migration80.incompatible.php#migration80.incompatible.tokenizer
+        if (defined('T_NAME_QUALIFIED') && $arrayToken[0] === T_NAME_QUALIFIED) {
+            return self::TOKEN_DEFAULT;
+        }
+        if (defined('T_NAME_FULLY_QUALIFIED') && $arrayToken[0] === T_NAME_FULLY_QUALIFIED) {
+            return self::TOKEN_DEFAULT;
+        }
+        if (defined('T_NAME_RELATIVE') && $arrayToken[0] === T_NAME_RELATIVE) {
+            return self::TOKEN_DEFAULT;
+        }
+
+        return self::TOKEN_KEYWORD;
+    }
+
     /**
      * @param array $tokens
      * @return array

tests/HighlighterTest.php

@@ -297,46 +297,67 @@ public function testFunctionCall()
 
     public function testFQNFunctionCall()
     {
-        $this->compare(
-            <<<'EOL'
+        $original = <<<'EOL'
 <?php
 echo \My\Package\functionName();
-EOL
-            ,
-            <<<'EOL'
+EOL;
+
+        if (PHP_VERSION_ID < 80000) {
+            $expected = <<<'EOL'
 <token_default><?php</token_default>
 <token_keyword>echo \</token_keyword><token_default>My</token_default><token_keyword>\</token_keyword><token_default>Package</token_default><token_keyword>\</token_keyword><token_default>functionName</token_default><token_keyword>();</token_keyword>
-EOL
-        );
+EOL;
+        } else {
+            $expected = <<<'EOL'
+<token_default><?php</token_default>
+<token_keyword>echo </token_keyword><token_default>\My\Package\functionName</token_default><token_keyword>();</token_keyword>
+EOL;
+        }
+
+        $this->compare($original, $expected);
     }
 
     public function testNamespaceRelativeFunctionCall()
     {
-        $this->compare(
-            <<<'EOL'
+        $original = <<<'EOL'
 <?php
 echo namespace\functionName();
-EOL
-            ,
-            <<<'EOL'
+EOL;
+
+        if (PHP_VERSION_ID < 80000) {
+            $expected = <<<'EOL'
 <token_default><?php</token_default>
 <token_keyword>echo namespace\</token_keyword><token_default>functionName</token_default><token_keyword>();</token_keyword>
-EOL
-        );
+EOL;
+        } else {
+            $expected = <<<'EOL'
+<token_default><?php</token_default>
+<token_keyword>echo </token_keyword><token_default>namespace\functionName</token_default><token_keyword>();</token_keyword>
+EOL;
+        }
+
+        $this->compare($original, $expected);
     }
 
     public function testQualifiedFunctionCall()
     {
-        $this->compare(
-            <<<'EOL'
+        $original = <<<'EOL'
 <?php
 echo Package\functionName();
-EOL
-            ,
-            <<<'EOL'
+EOL;
+
+        if (PHP_VERSION_ID < 80000) {
+            $expected = <<<'EOL'
 <token_default><?php</token_default>
 <token_keyword>echo </token_keyword><token_default>Package</token_default><token_keyword>\</token_keyword><token_default>functionName</token_default><token_keyword>();</token_keyword>
-EOL
-        );
+EOL;
+        } else {
+            $expected = <<<'EOL'
+<token_default><?php</token_default>
+<token_keyword>echo </token_keyword><token_default>Package\functionName</token_default><token_keyword>();</token_keyword>
+EOL;
+        }
+
+        $this->compare($original, $expected);
     }
 }