Add PhpToken class

RFC: https://wiki.php.net/rfc/token_as_object

Relative to the RFC, this also adds a __toString() method,
as discussed on list.

Closes GH-5176.

Author: nikic

UPGRADING

@@ -533,6 +533,12 @@ PHP 8.0 UPGRADE NOTES
 7. New Classes and Interfaces
 ========================================
 
+- Tokenizer:
+  . The new PhpToken class adds an object-based interface to the tokenizer.
+    It provides a more uniform and ergonomic representation, while being more
+    memory efficient and faster.
+    RFC: https://wiki.php.net/rfc/token_as_object
+
 ========================================
 8. Removed Extensions and SAPIs
 ========================================

ext/tokenizer/tests/PhpToken_constructor.phpt

@@ -0,0 +1,46 @@
+--TEST--
+PhpToken constructor
+--SKIPIF--
+<?php if (!extension_loaded("tokenizer")) print "skip tokenizer extension not enabled"; ?>
+--FILE--
+<?php
+
+$token = new PhpToken(300, 'function');
+var_dump($token);
+$token = new PhpToken(300, 'function', 10);
+var_dump($token);
+$token = new PhpToken(300, 'function', 10, 100);
+var_dump($token);
+
+?>
+--EXPECT--
+object(PhpToken)#1 (4) {
+  ["id"]=>
+  int(300)
+  ["text"]=>
+  string(8) "function"
+  ["line"]=>
+  int(-1)
+  ["pos"]=>
+  int(-1)
+}
+object(PhpToken)#2 (4) {
+  ["id"]=>
+  int(300)
+  ["text"]=>
+  string(8) "function"
+  ["line"]=>
+  int(10)
+  ["pos"]=>
+  int(-1)
+}
+object(PhpToken)#1 (4) {
+  ["id"]=>
+  int(300)
+  ["text"]=>
+  string(8) "function"
+  ["line"]=>
+  int(10)
+  ["pos"]=>
+  int(100)
+}

ext/tokenizer/tests/PhpToken_extension.phpt

@@ -0,0 +1,36 @@
+--TEST--
+Extending the PhpToken class
+--SKIPIF--
+<?php if (!extension_loaded("tokenizer")) print "skip tokenizer extension not enabled"; ?>
+--FILE--
+<?php
+
+$code = <<<'PHP'
+<?PHP
+FUNCTION FOO() {
+    ECHO "bar";
+}
+PHP;
+
+class MyPhpToken extends PhpToken {
+    public int $extra = 123;
+
+    public function getLoweredText(): string {
+        return strtolower($this->text);
+    }
+}
+
+foreach (MyPhpToken::getAll($code) as $token) {
+    echo $token->getLoweredText();
+
+    if ($token->extra !== 123) {
+        echo "Missing property!\n";
+    }
+}
+
+?>
+--EXPECT--
+<?php
+function foo() {
+    echo "bar";
+}

ext/tokenizer/tests/PhpToken_extension_errors.phpt

@@ -0,0 +1,30 @@
+--TEST--
+PhpToken extensions that throw during construction
+--SKIPIF--
+<?php if (!extension_loaded("tokenizer")) print "skip tokenizer extension not enabled"; ?>
+--FILE--
+<?php
+
+class MyPhpToken1 extends PhpToken {
+    public $extra = UNKNOWN;
+}
+
+try {
+    var_dump(MyPhpToken1::getAll("<?php foo"));
+} catch (Error $e) {
+    echo $e->getMessage(), "\n";
+}
+
+abstract class MyPhpToken2 extends PhpToken {
+}
+
+try {
+    var_dump(MyPhpToken2::getAll("<?php foo"));
+} catch (Error $e) {
+    echo $e->getMessage(), "\n";
+}
+
+?>
+--EXPECT--
+Undefined constant 'UNKNOWN'
+Cannot instantiate abstract class MyPhpToken2

ext/tokenizer/tests/PhpToken_final_constructor.phpt

@@ -0,0 +1,15 @@
+--TEST--
+Check that the PhpToken constructor is final
+--SKIPIF--
+<?php if (!extension_loaded("tokenizer")) print "skip tokenizer extension not enabled"; ?>
+--FILE--
+<?php
+
+class MyPhpToken extends PhpToken {
+    public function __construct() {
+    }
+}
+
+?>
+--EXPECTF--
+Fatal error: Cannot override final method PhpToken::__construct() in %s on line %d