From 8ab4ec53b2c06687bd7f1949024b68b08364cccd Mon Sep 17 00:00:00 2001
From: Vincent Langlet <vincentlanglet@hotmail.fr>
Date: Fri, 24 May 2019 06:15:22 +0200
Subject: [PATCH] :bug: Bugfix

---
 TwigCS/Command/TwigCSCommand.php              |   3 +-
 TwigCS/Config/Config.php                      |   7 +-
 TwigCS/Environment/StubbedEnvironment.php     |  16 +-
 TwigCS/Linter.php                             |  29 ++--
 TwigCS/Report/SniffViolation.php              |   6 +-
 TwigCS/Report/TextFormatter.php               |  39 +++--
 TwigCS/Ruleset/Ruleset.php                    |   5 +-
 TwigCS/Ruleset/RulesetFactory.php             |   6 +-
 TwigCS/Sniff/AbstractPostParserSniff.php      |   7 +-
 TwigCS/Sniff/AbstractPreParserSniff.php       |   3 +
 TwigCS/Sniff/AbstractSniff.php                |   5 +-
 TwigCS/Sniff/SniffInterface.php               |   3 +-
 .../Standard/DisallowCommentedCodeSniff.php   |   3 +
 .../Sniff/Standard/EnsureBlankAtEOFSniff.php  |   3 +
 TwigCS/Tests/AbstractSniffTest.php            |  14 +-
 TwigCS/Token/TokenParser.php                  |   5 +
 TwigCS/Token/Tokenizer.php                    | 113 +++++++-------
 TwigCS/Token/TokenizerHelper.php              | 145 ++++--------------
 TwigCS/Token/TokenizerInterface.php           |  18 ---
 composer.json                                 |   6 +-
 composer.lock                                 | 100 +++++++++++-
 21 files changed, 303 insertions(+), 233 deletions(-)
 delete mode 100644 TwigCS/Token/TokenizerInterface.php

diff --git a/TwigCS/Command/TwigCSCommand.php b/TwigCS/Command/TwigCSCommand.php
index c3bcae2..d8b472b 100644
--- a/TwigCS/Command/TwigCSCommand.php
+++ b/TwigCS/Command/TwigCSCommand.php
@@ -2,6 +2,7 @@
 
 namespace TwigCS\Command;
 
+use \Exception;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
@@ -65,7 +66,7 @@ protected function configure()
      *
      * @return int
      *
-     * @throws \Exception
+     * @throws Exception
      */
     protected function execute(InputInterface $input, OutputInterface $output)
     {
diff --git a/TwigCS/Config/Config.php b/TwigCS/Config/Config.php
index 1aa5095..00bca4f 100644
--- a/TwigCS/Config/Config.php
+++ b/TwigCS/Config/Config.php
@@ -2,6 +2,7 @@
 
 namespace TwigCS\Config;
 
+use \Exception;
 use Symfony\Component\Finder\Finder;
 
 /**
@@ -43,6 +44,8 @@ public function __construct()
      * Find all files to process, based on a file or directory and exclude patterns.
      *
      * @return array
+     *
+     * @throws Exception
      */
     public function findFiles()
     {
@@ -74,11 +77,13 @@ public function findFiles()
      * @param string $key
      *
      * @return mixed
+     *
+     * @throws Exception
      */
     public function get($key)
     {
         if (!isset($this->config[$key])) {
-            throw new \Exception(sprintf('Configuration key "%s" does not exist', $key));
+            throw new Exception(sprintf('Configuration key "%s" does not exist', $key));
         }
 
         return $this->config[$key];
diff --git a/TwigCS/Environment/StubbedEnvironment.php b/TwigCS/Environment/StubbedEnvironment.php
index 054901a..53596ae 100644
--- a/TwigCS/Environment/StubbedEnvironment.php
+++ b/TwigCS/Environment/StubbedEnvironment.php
@@ -2,6 +2,13 @@
 
 namespace TwigCS\Environment;
 
+use \Closure;
+use Symfony\Bridge\Twig\TokenParser\DumpTokenParser;
+use Symfony\Bridge\Twig\TokenParser\FormThemeTokenParser;
+use Symfony\Bridge\Twig\TokenParser\StopwatchTokenParser;
+use Symfony\Bridge\Twig\TokenParser\TransChoiceTokenParser;
+use Symfony\Bridge\Twig\TokenParser\TransDefaultDomainTokenParser;
+use Symfony\Bridge\Twig\TokenParser\TransTokenParser;
 use Twig\Environment;
 use Twig\Loader\LoaderInterface;
 use Twig\TwigFilter;
@@ -31,7 +38,7 @@ class StubbedEnvironment extends Environment
     private $stubTests;
 
     /**
-     * @var \Closure
+     * @var Closure
      */
     private $stubCallable;
 
@@ -43,6 +50,13 @@ public function __construct(LoaderInterface $loader = null, $options = [])
     {
         parent::__construct($loader, $options);
 
+        $this->addTokenParser(new DumpTokenParser());
+        $this->addTokenParser(new FormThemeTokenParser());
+        $this->addTokenParser(new StopwatchTokenParser(false));
+        $this->addTokenParser(new TransChoiceTokenParser());
+        $this->addTokenParser(new TransDefaultDomainTokenParser());
+        $this->addTokenParser(new TransTokenParser());
+
         $this->stubCallable  = function () {
             /* This will be used as stub filter, function or test */
         };
diff --git a/TwigCS/Linter.php b/TwigCS/Linter.php
index b848953..cbb15a3 100644
--- a/TwigCS/Linter.php
+++ b/TwigCS/Linter.php
@@ -2,6 +2,8 @@
 
 namespace TwigCS;
 
+use \Exception;
+use \Traversable;
 use Twig\Environment;
 use Twig\Error\Error;
 use Twig\Source;
@@ -12,7 +14,7 @@
 use TwigCS\Sniff\PostParserSniffInterface;
 use TwigCS\Sniff\PreParserSniffInterface;
 use TwigCS\Sniff\SniffInterface;
-use TwigCS\Token\TokenizerInterface;
+use TwigCS\Token\Tokenizer;
 
 /**
  * Linter is the main class and will process twig files against a set of rules.
@@ -30,17 +32,18 @@ class Linter
     protected $sniffsExtension;
 
     /**
-     * @var TokenizerInterface
+     * @var Tokenizer
      */
     protected $tokenizer;
 
     /**
-     * @param Environment        $env
-     * @param TokenizerInterface $tokenizer
+     * @param Environment $env
+     * @param Tokenizer   $tokenizer
      */
-    public function __construct(Environment $env, TokenizerInterface $tokenizer)
+    public function __construct(Environment $env, Tokenizer $tokenizer)
     {
         $this->env = $env;
+
         $this->sniffsExtension = $this->env->getExtension('TwigCS\Extension\SniffsExtension');
         $this->tokenizer = $tokenizer;
     }
@@ -52,15 +55,17 @@ public function __construct(Environment $env, TokenizerInterface $tokenizer)
      * @param Ruleset      $ruleset Set of rules to check.
      *
      * @return Report an object with all violations and stats.
+     *
+     * @throws Exception
      */
     public function run($files, Ruleset $ruleset)
     {
-        if (!is_array($files) && !$files instanceof \Traversable) {
+        if (!is_array($files) && !$files instanceof Traversable) {
             $files = [$files];
         }
 
         if (empty($files)) {
-            throw new \Exception('No files to process, provide at least one file to be linted');
+            throw new Exception('No files to process, provide at least one file to be linted');
         }
 
         // setUp
@@ -70,8 +75,8 @@ public function run($files, Ruleset $ruleset)
                 $sniffViolation = new SniffViolation(
                     SniffInterface::MESSAGE_TYPE_NOTICE,
                     $msg,
-                    '',
-                    ''
+                    null,
+                    null
                 );
 
                 $report->addMessage($sniffViolation);
@@ -139,11 +144,11 @@ public function processTemplate($file, $ruleset, $report)
         // Tokenizer.
         try {
             $stream = $this->tokenizer->tokenize($twigSource);
-        } catch (\Exception $e) {
+        } catch (Exception $e) {
             $sniffViolation = new SniffViolation(
                 SniffInterface::MESSAGE_TYPE_ERROR,
-                sprintf('Unable to tokenize file "%s"', (string) $file),
-                '',
+                sprintf('Unable to tokenize file'),
+                null,
                 (string) $file
             );
 
diff --git a/TwigCS/Report/SniffViolation.php b/TwigCS/Report/SniffViolation.php
index 4433d90..4f0b6a2 100644
--- a/TwigCS/Report/SniffViolation.php
+++ b/TwigCS/Report/SniffViolation.php
@@ -91,10 +91,9 @@ public function getLevelAsString()
             case SniffInterface::MESSAGE_TYPE_WARNING:
                 return 'WARNING';
             case SniffInterface::MESSAGE_TYPE_ERROR:
+            default:
                 return 'ERROR';
         }
-
-        throw new \Exception(sprintf('Unknown level "%s"', $this->level));
     }
 
     /**
@@ -112,10 +111,9 @@ public static function getLevelAsInt($level)
             case 'WARNING':
                 return SniffInterface::MESSAGE_TYPE_WARNING;
             case 'ERROR':
+            default:
                 return SniffInterface::MESSAGE_TYPE_ERROR;
         }
-
-        throw new \Exception(sprintf('Unknown level "%s"', $level));
     }
 
     /**
diff --git a/TwigCS/Report/TextFormatter.php b/TwigCS/Report/TextFormatter.php
index 3f6f244..ed765ab 100644
--- a/TwigCS/Report/TextFormatter.php
+++ b/TwigCS/Report/TextFormatter.php
@@ -46,33 +46,40 @@ public function display(Report $report, $level = null)
                 'level' => $level,
             ]);
 
-            $this->io->text((count($fileMessages) > 0 ? '<fg=red>KO</fg=red>' : '<info>OK</info>').' '.$file);
+            if (count($fileMessages) > 0) {
+                $this->io->text('<fg=red>KO</fg=red> '.$file);
+            }
 
             $rows = [];
             foreach ($fileMessages as $message) {
                 $lines = $this->getContext(file_get_contents($file), $message->getLine(), $this::ERROR_CONTEXT_LIMIT);
 
                 $formattedText = [];
+                if (!$message->getLine()) {
+                    $formattedText[] = $this->formatErrorMessage($message);
+                }
+
                 foreach ($lines as $no => $code) {
                     $formattedText[] = sprintf($this::ERROR_LINE_FORMAT, $no, wordwrap($code, $this::ERROR_LINE_WIDTH));
 
                     if ($no === $message->getLine()) {
-                        $formattedText[] = sprintf(
-                            '<fg=red>'.$this::ERROR_LINE_FORMAT.'</fg=red>',
-                            $this::ERROR_CURSOR_CHAR,
-                            wordwrap($message->getMessage(), $this::ERROR_LINE_WIDTH)
-                        );
+                        $formattedText[] = $this->formatErrorMessage($message);
                     }
                 }
 
+                if (count($rows) > 0) {
+                    $rows[] = new TableSeparator();
+                }
+
                 $rows[] = [
-                    new TableCell('<comment>'.$message->getLevelAsString().'</comment>', ['rowspan' => 2]),
+                    new TableCell('<comment>'.$message->getLevelAsString().'</comment>'),
                     implode("\n", $formattedText),
                 ];
-                $rows[] = new TableSeparator();
             }
 
-            $this->io->table([], $rows);
+            if (count($rows) > 0) {
+                $this->io->table([], $rows);
+            }
         }
 
         $summaryString = sprintf(
@@ -131,4 +138,18 @@ protected function getContext($template, $line, $context)
 
         return $result;
     }
+
+    /**
+     * @param SniffViolation $message
+     *
+     * @return string
+     */
+    protected function formatErrorMessage(SniffViolation $message)
+    {
+        return sprintf(
+            '<fg=red>'.$this::ERROR_LINE_FORMAT.'</fg=red>',
+            $this::ERROR_CURSOR_CHAR,
+            wordwrap($message->getMessage(), $this::ERROR_LINE_WIDTH)
+        );
+    }
 }
diff --git a/TwigCS/Ruleset/Ruleset.php b/TwigCS/Ruleset/Ruleset.php
index 719bd1f..485b044 100644
--- a/TwigCS/Ruleset/Ruleset.php
+++ b/TwigCS/Ruleset/Ruleset.php
@@ -2,6 +2,7 @@
 
 namespace TwigCS\Ruleset;
 
+use \Exception;
 use TwigCS\Sniff\PostParserSniffInterface;
 use TwigCS\Sniff\PreParserSniffInterface;
 use TwigCS\Sniff\SniffInterface;
@@ -69,6 +70,8 @@ public function addPostParserSniff(PostParserSniffInterface $sniff)
      * @param SniffInterface $sniff
      *
      * @return $this
+     *
+     * @throws Exception
      */
     public function addSniff(SniffInterface $sniff)
     {
@@ -88,7 +91,7 @@ public function addSniff(SniffInterface $sniff)
             return $this;
         }
 
-        throw new \Exception(sprintf(
+        throw new Exception(sprintf(
             'Unknown type of sniff "%s", expected one of: "%s"',
             $sniff->getType(),
             implode(', ', [SniffInterface::TYPE_PRE_PARSER, SniffInterface::TYPE_POST_PARSER])
diff --git a/TwigCS/Ruleset/RulesetFactory.php b/TwigCS/Ruleset/RulesetFactory.php
index 8d62322..b85ebfc 100644
--- a/TwigCS/Ruleset/RulesetFactory.php
+++ b/TwigCS/Ruleset/RulesetFactory.php
@@ -2,6 +2,8 @@
 
 namespace TwigCS\Ruleset;
 
+use \Exception;
+use \SplFileInfo;
 use Symfony\Component\Finder\Finder;
 
 /**
@@ -13,6 +15,8 @@ class RulesetFactory
      * Create a new set of rule.
      *
      * @return Ruleset
+     *
+     * @throws Exception
      */
     public function createStandardRuleset()
     {
@@ -20,7 +24,7 @@ public function createStandardRuleset()
 
         $finder = Finder::create()->in(__DIR__.'/../Sniff/Standard')->files();
 
-        /** @var \SplFileInfo $file */
+        /** @var SplFileInfo $file */
         foreach ($finder as $file) {
             $class = 'TwigCS\Sniff\Standard\\'.explode('.', $file->getFilename())[0];
             $ruleset->addSniff(new $class());
diff --git a/TwigCS/Sniff/AbstractPostParserSniff.php b/TwigCS/Sniff/AbstractPostParserSniff.php
index e040317..df6ad59 100644
--- a/TwigCS/Sniff/AbstractPostParserSniff.php
+++ b/TwigCS/Sniff/AbstractPostParserSniff.php
@@ -2,6 +2,7 @@
 
 namespace TwigCS\Sniff;
 
+use \Exception;
 use Twig\Node\Expression\Binary\ConcatBinary;
 use Twig\Node\Expression\ConstantExpression;
 use Twig\Node\Expression\FilterExpression;
@@ -38,6 +39,8 @@ public function getType()
      * @param Node   $node
      *
      * @return self
+     *
+     * @throws Exception
      */
     public function addMessage($messageType, $message, Node $node)
     {
@@ -56,7 +59,7 @@ public function addMessage($messageType, $message, Node $node)
     /**
      * @param Node $node
      *
-     * @return string
+     * @return int|null
      */
     public function getTemplateLine(Node $node)
     {
@@ -68,7 +71,7 @@ public function getTemplateLine(Node $node)
             return $node->getLine();
         }
 
-        return '';
+        return null;
     }
 
     /**
diff --git a/TwigCS/Sniff/AbstractPreParserSniff.php b/TwigCS/Sniff/AbstractPreParserSniff.php
index bfe8c40..0ac7739 100644
--- a/TwigCS/Sniff/AbstractPreParserSniff.php
+++ b/TwigCS/Sniff/AbstractPreParserSniff.php
@@ -2,6 +2,7 @@
 
 namespace TwigCS\Sniff;
 
+use \Exception;
 use TwigCS\Report\SniffViolation;
 use TwigCS\Token\Token;
 
@@ -46,6 +47,8 @@ public function isTokenMatching(Token $token, $type, $value = null)
      * @param Token  $token
      *
      * @return self
+     *
+     * @throws Exception
      */
     public function addMessage($messageType, $message, Token $token)
     {
diff --git a/TwigCS/Sniff/AbstractSniff.php b/TwigCS/Sniff/AbstractSniff.php
index 242672a..0b58109 100644
--- a/TwigCS/Sniff/AbstractSniff.php
+++ b/TwigCS/Sniff/AbstractSniff.php
@@ -2,6 +2,7 @@
 
 namespace TwigCS\Sniff;
 
+use \Exception;
 use TwigCS\Report\Report;
 
 /**
@@ -51,11 +52,13 @@ public function disable()
 
     /**
      * @return Report
+     *
+     * @throws Exception
      */
     public function getReport()
     {
         if (null === $this->report) {
-            throw new \Exception('Sniff is disabled!');
+            throw new Exception('Sniff is disabled!');
         }
 
         return $this->report;
diff --git a/TwigCS/Sniff/SniffInterface.php b/TwigCS/Sniff/SniffInterface.php
index 5a21594..37571cb 100644
--- a/TwigCS/Sniff/SniffInterface.php
+++ b/TwigCS/Sniff/SniffInterface.php
@@ -2,6 +2,7 @@
 
 namespace TwigCS\Sniff;
 
+use \Exception;
 use TwigCS\Report\Report;
 
 /**
@@ -42,7 +43,7 @@ public function disable();
      *
      * @return Report
      *
-     * @throws \Exception A disabled sniff has no current report.
+     * @throws Exception A disabled sniff has no current report.
      */
     public function getReport();
 
diff --git a/TwigCS/Sniff/Standard/DisallowCommentedCodeSniff.php b/TwigCS/Sniff/Standard/DisallowCommentedCodeSniff.php
index 99b90e2..3845c3c 100644
--- a/TwigCS/Sniff/Standard/DisallowCommentedCodeSniff.php
+++ b/TwigCS/Sniff/Standard/DisallowCommentedCodeSniff.php
@@ -2,6 +2,7 @@
 
 namespace TwigCS\Sniff\Standard;
 
+use \Exception;
 use TwigCS\Sniff\AbstractPreParserSniff;
 use TwigCS\Token\Token;
 
@@ -18,6 +19,8 @@ class DisallowCommentedCodeSniff extends AbstractPreParserSniff
      * @param Token[] $tokens
      *
      * @return Token
+     *
+     * @throws Exception
      */
     public function process(Token $token, $tokenPosition, $tokens)
     {
diff --git a/TwigCS/Sniff/Standard/EnsureBlankAtEOFSniff.php b/TwigCS/Sniff/Standard/EnsureBlankAtEOFSniff.php
index 8106059..cbbfc81 100644
--- a/TwigCS/Sniff/Standard/EnsureBlankAtEOFSniff.php
+++ b/TwigCS/Sniff/Standard/EnsureBlankAtEOFSniff.php
@@ -2,6 +2,7 @@
 
 namespace TwigCS\Sniff\Standard;
 
+use \Exception;
 use TwigCS\Sniff\AbstractPreParserSniff;
 use TwigCS\Token\Token;
 
@@ -16,6 +17,8 @@ class EnsureBlankAtEOFSniff extends AbstractPreParserSniff
      * @param Token[] $tokens
      *
      * @return Token
+     *
+     * @throws Exception
      */
     public function process(Token $token, $tokenPosition, $tokens)
     {
diff --git a/TwigCS/Tests/AbstractSniffTest.php b/TwigCS/Tests/AbstractSniffTest.php
index 7585dc3..5701d16 100644
--- a/TwigCS/Tests/AbstractSniffTest.php
+++ b/TwigCS/Tests/AbstractSniffTest.php
@@ -2,6 +2,7 @@
 
 namespace TwigCS\Tests;
 
+use \Exception;
 use PHPUnit\Framework\MockObject\MockObject;
 use PHPUnit\Framework\TestCase;
 use Twig\Loader\LoaderInterface;
@@ -34,7 +35,7 @@ public function setUp()
         $this->env = new StubbedEnvironment(
             $twigLoaderInterface,
             [
-                'stub_tags'  => ['dump', 'render', 'some_other_block', 'stylesheets', 'trans'],
+                'stub_tags'  => ['render', 'some_other_block', 'stylesheets'],
                 'stub_tests' => ['some_test'],
             ]
         );
@@ -45,17 +46,20 @@ public function setUp()
      * @param string         $filename
      * @param SniffInterface $sniff
      * @param array          $expects
-     *
-     * @throws \Exception
      */
     protected function checkGenericSniff($filename, SniffInterface $sniff, array $expects)
     {
         $file = __DIR__.'/Fixtures/'.$filename;
 
         $ruleset = new Ruleset();
-        $ruleset->addSniff($sniff);
+        try {
+            $ruleset->addSniff($sniff);
+            $report = $this->lint->run($file, $ruleset);
+        } catch (Exception $e) {
+            $this->fail($e->getMessage());
 
-        $report = $this->lint->run($file, $ruleset);
+            return;
+        }
 
         $this->assertEquals(count($expects), $report->getTotalWarnings() + $report->getTotalErrors());
         if ($expects) {
diff --git a/TwigCS/Token/TokenParser.php b/TwigCS/Token/TokenParser.php
index 543633d..afe83a8 100644
--- a/TwigCS/Token/TokenParser.php
+++ b/TwigCS/Token/TokenParser.php
@@ -2,6 +2,7 @@
 
 namespace TwigCS\Token;
 
+use \Exception;
 use Twig\Node\Node;
 use Twig\Token;
 use Twig\TokenParser\AbstractTokenParser;
@@ -39,6 +40,8 @@ public function decideEnd(Token $token)
      * @param Token $token
      *
      * @return Node
+     *
+     * @throws Exception
      */
     public function parse(Token $token)
     {
@@ -75,6 +78,8 @@ public function getTag()
      * @param TokenStream $stream
      *
      * @return bool
+     *
+     * @throws Exception
      */
     private function hasBody(TokenStream $stream)
     {
diff --git a/TwigCS/Token/Tokenizer.php b/TwigCS/Token/Tokenizer.php
index 28b1076..68aa19d 100644
--- a/TwigCS/Token/Tokenizer.php
+++ b/TwigCS/Token/Tokenizer.php
@@ -2,6 +2,7 @@
 
 namespace TwigCS\Token;
 
+use \Exception;
 use Twig\Environment;
 use Twig\Lexer;
 use Twig\Source;
@@ -9,7 +10,7 @@
 /**
  * An override of Twig's Lexer to add whitespace and new line detection.
  */
-class Tokenizer implements TokenizerInterface
+class Tokenizer
 {
     const STATE_DATA          = Lexer::STATE_DATA;
     const STATE_BLOCK         = Lexer::STATE_BLOCK;
@@ -92,27 +93,20 @@ class Tokenizer implements TokenizerInterface
     public function __construct(Environment $env, array $options = [])
     {
         $this->options = array_merge([
-            'tag_comment'           => ['{#', '#}'],
-            'tag_block'             => ['{%', '%}'],
-            'tag_variable'          => ['{{', '}}'],
-            'whitespace_trim'       => '-',
-            'whitespace_line_trim'  => '~',
-            'whitespace_line_chars' => ' \t\0\x0B',
-            'interpolation'         => ['#{', '}'],
+            'tag_comment'     => ['{#', '#}'],
+            'tag_block'       => ['{%', '%}'],
+            'tag_variable'    => ['{{', '}}'],
+            'whitespace_trim' => '-',
+            'interpolation'   => ['#{', '}'],
         ], $options);
 
         $tokenizerHelper = new TokenizerHelper($env, $this->options);
         $this->regexes = [
-            'lex_var'             => $tokenizerHelper->getVarRegex(),
-            'lex_block'           => $tokenizerHelper->getBlockRegex(),
-            'lex_raw_data'        => $tokenizerHelper->getRawDataRegex(),
-            'operator'            => $tokenizerHelper->getOperatorRegex(),
-            'lex_comment'         => $tokenizerHelper->getCommentRegex(),
-            'lex_block_raw'       => $tokenizerHelper->getBlockRawRegex(),
-            'lex_block_line'      => $tokenizerHelper->getBlockLineRegex(),
-            'lex_tokens_start'    => $tokenizerHelper->getTokensStartRegex(),
-            'interpolation_start' => $tokenizerHelper->getInterpolationStartRegex(),
-            'interpolation_end'   => $tokenizerHelper->getInterpolationEndRegex(),
+            'lex_block'        => $tokenizerHelper->getBlockRegex(),
+            'lex_comment'      => $tokenizerHelper->getCommentRegex(),
+            'lex_variable'     => $tokenizerHelper->getVariableRegex(),
+            'operator'         => $tokenizerHelper->getOperatorRegex(),
+            'lex_tokens_start' => $tokenizerHelper->getTokensStartRegex(),
         ];
     }
 
@@ -120,6 +114,8 @@ public function __construct(Environment $env, array $options = [])
      * @param Source $source
      *
      * @return Token[]
+     *
+     * @throws Exception
      */
     public function tokenize(Source $source)
     {
@@ -127,7 +123,14 @@ public function tokenize(Source $source)
         $this->preflightSource($this->code);
 
         while ($this->cursor < $this->end) {
-            $nextToken = $this->getTokenPosition();
+            $lastToken = $this->getTokenPosition();
+            $nextToken = $this->getTokenPosition(1);
+
+            while (null !== $nextToken && $nextToken['position'] < $this->cursor) {
+                $this->moveCurrentPosition();
+                $lastToken = $nextToken;
+                $nextToken = $this->getTokenPosition(1);
+            }
 
             switch ($this->getState()) {
                 case self::STATE_BLOCK:
@@ -139,26 +142,20 @@ public function tokenize(Source $source)
                 case self::STATE_COMMENT:
                     $this->lexComment();
                     break;
-//                case self::STATE_STRING:
-//                    $this->lexString();
-//                    break;
-//                case self::STATE_INTERPOLATION:
-//                    $this->lexInterpolation();
-//                    break;
                 case self::STATE_DATA:
-                    if ($this->cursor === $nextToken['position']) {
+                    if (null !== $lastToken && $this->cursor === $lastToken['position']) {
                         $this->lexStart();
                     } else {
                         $this->lexData();
                     }
                     break;
                 default:
-                    throw new \Exception('Unhandled state in tokenize', 1);
+                    throw new Exception('Unhandled state in tokenize', 1);
             }
         }
 
         if (self::STATE_DATA !== $this->getState()) {
-            throw new \Exception('Error Processing Request', 1);
+            throw new Exception('Error Processing Request', 1);
         }
 
         $this->pushToken(Token::EOF_TYPE);
@@ -178,7 +175,7 @@ protected function resetState(Source $source)
         $this->state = [];
 
         $this->code = str_replace(["\r\n", "\r"], "\n", $source->getCode());
-        $this->end = \strlen($this->code);
+        $this->end = strlen($this->code);
         $this->filename = $source->getName();
     }
 
@@ -199,12 +196,12 @@ protected function pushState($state)
     }
 
     /**
-     * @throws \Exception
+     * @throws Exception
      */
     protected function popState()
     {
         if (0 === count($this->state)) {
-            throw new \Exception('Cannot pop state without a previous state');
+            throw new Exception('Cannot pop state without a previous state');
         }
         array_pop($this->state);
     }
@@ -230,15 +227,19 @@ protected function preflightSource($code)
     }
 
     /**
+     * @param int $offset
+     *
      * @return array|null
      */
-    protected function getTokenPosition()
+    protected function getTokenPosition($offset = 0)
     {
-        if (empty($this->tokenPositions) || !isset($this->tokenPositions[$this->currentPosition])) {
+        if (empty($this->tokenPositions)
+            || !isset($this->tokenPositions[$this->currentPosition + $offset])
+        ) {
             return null;
         }
 
-        return $this->tokenPositions[$this->currentPosition];
+        return $this->tokenPositions[$this->currentPosition + $offset];
     }
 
     /**
@@ -272,39 +273,38 @@ protected function pushToken($type, $value = null)
      * @param int    $endType
      * @param string $endRegex
      *
-     * @throws \Exception
+     * @throws Exception
      */
     protected function lex($endType, $endRegex)
     {
         preg_match($endRegex, $this->code, $match, PREG_OFFSET_CAPTURE, $this->cursor);
+
         if (!isset($match[0])) {
-            // Should not happen, but in case it is;
-            throw new \Exception(sprintf('Unclosed "%s" in "%s" at line %d', $endType, $this->filename, $this->line));
-        }
-        if ($match[0][1] === $this->cursor) {
+            $this->lexExpression();
+        } elseif ($match[0][1] === $this->cursor) {
             $this->pushToken($endType, $match[0][0]);
             $this->moveCursor($match[0][0]);
             $this->moveCurrentPosition();
             $this->popState();
+        } elseif ($this->getState() === self::STATE_COMMENT) {
+            // Parse as text until the end position.
+            $this->lexData($match[0][1]);
         } else {
-            if ($this->getState() === self::STATE_COMMENT) {
-                // Parse as text until the end position.
-                $this->lexData($match[0][1]);
-            } else {
-                while ($this->cursor < $match[0][1]) {
-                    $this->lexExpression();
-                }
+            while ($this->cursor < $match[0][1]) {
+                $this->lexExpression();
             }
         }
     }
 
     /**
-     * @throws \Exception
+     * @throws Exception
      */
     protected function lexExpression()
     {
         $currentToken = $this->code[$this->cursor];
-        if (' ' === $currentToken) {
+        if (preg_match('/\t/', $currentToken)) {
+            $this->lexTab();
+        } elseif (' ' === $currentToken) {
             $this->lexWhitespace();
         } elseif (PHP_EOL === $currentToken) {
             $this->lexEOL();
@@ -332,11 +332,11 @@ protected function lexExpression()
             } elseif (false !== strpos(')]}', $this->code[$this->cursor])) {
                 // closing bracket
                 if (empty($this->brackets)) {
-                    throw new \Exception(sprintf('Unexpected "%s".', $this->code[$this->cursor]));
+                    throw new Exception(sprintf('Unexpected "%s"', $this->code[$this->cursor]));
                 }
                 $expect = array_pop($this->brackets)[0];
                 if (strtr($expect, '([{', ')]}') !== $this->code[$this->cursor]) {
-                    throw new \Exception(sprintf('Unclosed "%s".', $expect));
+                    throw new Exception(sprintf('Unclosed "%s"', $expect));
                 }
             }
             $this->pushToken(Token::PUNCTUATION_TYPE, $this->code[$this->cursor]);
@@ -347,12 +347,12 @@ protected function lexExpression()
             $this->moveCursor($match[0]);
         } else {
             // unlexable
-            throw new \Exception(sprintf('Unexpected character "%s".', $this->code[$this->cursor]));
+            throw new Exception(sprintf('Unexpected character "%s"', $currentToken));
         }
     }
 
     /**
-     * @throws \Exception
+     * @throws Exception
      */
     protected function lexBlock()
     {
@@ -360,7 +360,7 @@ protected function lexBlock()
     }
 
     /**
-     * @throws \Exception
+     * @throws Exception
      */
     protected function lexVariable()
     {
@@ -368,7 +368,7 @@ protected function lexVariable()
     }
 
     /**
-     * @throws \Exception
+     * @throws Exception
      */
     protected function lexComment()
     {
@@ -384,6 +384,7 @@ protected function lexData($limit = 0)
         if (0 === $limit && null !== $nextToken) {
             $limit = $nextToken['position'];
         }
+
         $currentToken = $this->code[$this->cursor];
         if (preg_match('/\t/', $currentToken)) {
             $this->lexTab();
@@ -408,7 +409,7 @@ protected function lexData($limit = 0)
     }
 
     /**
-     * @throws \Exception
+     * @throws Exception
      */
     protected function lexStart()
     {
@@ -423,7 +424,7 @@ protected function lexStart()
             $state = self::STATE_VAR;
             $tokenType = Token::VAR_START_TYPE;
         } else {
-            throw new \Exception(sprintf('Unhandled tag "%s" in lexStart', $tokenStart['match']), 1);
+            throw new Exception(sprintf('Unhandled tag "%s" in lexStart', $tokenStart['match']), 1);
         }
 
         $this->pushToken($tokenType, $tokenStart['fullMatch']);
diff --git a/TwigCS/Token/TokenizerHelper.php b/TwigCS/Token/TokenizerHelper.php
index a9c4793..b8e939d 100644
--- a/TwigCS/Token/TokenizerHelper.php
+++ b/TwigCS/Token/TokenizerHelper.php
@@ -32,57 +32,49 @@ public function __construct(Environment $env, array $options = [])
     /**
      * @return string
      */
-    public function getVarRegex()
+    public function getBlockRegex()
     {
-        return '{
-            \s*
-            (?:'.
-            preg_quote($this->options['whitespace_trim'].$this->options['tag_variable'][1], '#').'\s*'.
-            '|'.
-            preg_quote($this->options['whitespace_line_trim'].$this->options['tag_variable'][1], '#').'['.$this->options['whitespace_line_chars'].']*'.
-            '|'.
-            preg_quote($this->options['tag_variable'][1], '#').
-            ')
-        }Ax';
+        return '/'
+            .'('.preg_quote($this->options['whitespace_trim']).')?'
+            .'('.preg_quote($this->options['tag_block'][1]).')'
+            .'/A';
     }
 
     /**
      * @return string
      */
-    public function getBlockRegex()
+    public function getCommentRegex()
+    {
+        return '/'
+            .'('.preg_quote($this->options['whitespace_trim']).')?'
+            .'('.preg_quote($this->options['tag_comment'][1]).')'
+            .'/';
+    }
+
+    /**
+     * @return string
+     */
+    public function getVariableRegex()
     {
-        return '{
-            \s*
-            (?:'.
-            preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '#').'\s*\n?'.
-            '|'.
-            preg_quote($this->options['whitespace_line_trim'].$this->options['tag_block'][1], '#').'['.$this->options['whitespace_line_chars'].']*'.
-            '|'.
-            preg_quote($this->options['tag_block'][1], '#').'\n?'.
-            ')
-        }Ax';
+        return '/'
+            .'('.preg_quote($this->options['whitespace_trim']).')?'
+            .'('.preg_quote($this->options['tag_variable'][1]).')'
+            .'/A';
     }
 
     /**
      * @return string
      */
-    public function getRawDataRegex()
+    public function getTokensStartRegex()
     {
-        return '{'.
-            preg_quote($this->options['tag_block'][0], '#').
-            '('.
-            $this->options['whitespace_trim'].
-            '|'.
-            $this->options['whitespace_line_trim'].
-            ')?\s*endverbatim\s*'.
-            '(?:'.
-            preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '#').'\s*'.
-            '|'.
-            preg_quote($this->options['whitespace_line_trim'].$this->options['tag_block'][1], '#').'['.$this->options['whitespace_line_chars'].']*'.
-            '|'.
-            preg_quote($this->options['tag_block'][1], '#').
-            ')
-        }sx';
+        return '/'
+            .'('
+                .preg_quote($this->options['tag_variable'][0], '/')
+                .'|'.preg_quote($this->options['tag_block'][0], '/')
+                .'|'.preg_quote($this->options['tag_comment'][0], '/')
+            .')'
+            .'('.preg_quote($this->options['whitespace_trim'], '/').')?'
+            .'/s';
     }
 
     /**
@@ -114,81 +106,4 @@ public function getOperatorRegex()
 
         return '/'.implode('|', $regex).'/A';
     }
-
-    /**
-     * @return string
-     */
-    public function getCommentRegex()
-    {
-        return '{
-            (?:'.
-            preg_quote($this->options['whitespace_trim']).preg_quote($this->options['tag_comment'][1], '#').'\s*\n?'.
-            '|'.
-            preg_quote($this->options['whitespace_line_trim'].$this->options['tag_comment'][1], '#').'['.$this->options['whitespace_line_chars'].']*'.
-            '|'.
-            preg_quote($this->options['tag_comment'][1], '#').'\n?'.
-            ')
-        }sx';
-    }
-
-    /**
-     * @return string
-     */
-    public function getBlockRawRegex()
-    {
-        return '{
-            \s*verbatim\s*
-            (?:'.
-            preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '#').'\s*'.
-            '|'.
-            preg_quote($this->options['whitespace_line_trim'].$this->options['tag_block'][1], '#').'['.$this->options['whitespace_line_chars'].']*'.
-            '|'.
-            preg_quote($this->options['tag_block'][1], '#').
-            ')
-        }Asx';
-    }
-
-    /**
-     * @return string
-     */
-    public function getBlockLineRegex()
-    {
-        return '{\s*line\s+(\d+)\s*'.preg_quote($this->options['tag_block'][1], '#').'}As';
-    }
-
-    /**
-     * @return string
-     */
-    public function getTokensStartRegex()
-    {
-        return '{
-            ('.
-            preg_quote($this->options['tag_variable'][0], '#').
-            '|'.
-            preg_quote($this->options['tag_block'][0], '#').
-            '|'.
-            preg_quote($this->options['tag_comment'][0], '#').
-            ')('.
-            preg_quote($this->options['whitespace_trim'], '#').
-            '|'.
-            preg_quote($this->options['whitespace_line_trim'], '#').
-            ')?
-        }sx';
-    }
-
-    /**
-     * @return string
-     */
-    public function getInterpolationStartRegex()
-    {
-        return '{'.preg_quote($this->options['interpolation'][0], '#').'\s*}A';
-    }
-
-    /**
-     * @return string
-     */
-    public function getInterpolationEndRegex()
-    {
-        return '{\s*'.preg_quote($this->options['interpolation'][1], '#').'}A';
-    }
 }
diff --git a/TwigCS/Token/TokenizerInterface.php b/TwigCS/Token/TokenizerInterface.php
deleted file mode 100644
index d521b4a..0000000
--- a/TwigCS/Token/TokenizerInterface.php
+++ /dev/null
@@ -1,18 +0,0 @@
-<?php
-
-namespace TwigCS\Token;
-
-use Twig\Source;
-
-/**
- * Interface TokenizerInterface
- */
-interface TokenizerInterface
-{
-    /**
-     * @param Source $code
-     *
-     * @return Token[]
-     */
-    public function tokenize(Source $code);
-}
diff --git a/composer.json b/composer.json
index 88cf26a..245c639 100644
--- a/composer.json
+++ b/composer.json
@@ -17,10 +17,12 @@
     ],
     "bin": ["bin/twigcs"],
     "require": {
+        "php": ">=7.0",
         "squizlabs/php_codesniffer": "3.4.*",
-        "twig/twig": "^2.0",
         "symfony/console": "^3.0 || ^4.0",
-        "symfony/finder": "^3.0 || ^4.0"
+        "symfony/finder": "^3.0 || ^4.0",
+        "symfony/twig-bridge": "^3.0 || ^4.0",
+        "twig/twig": "^2.0"
     },
     "require-dev": {
         "phpunit/phpunit": "^7.0"
diff --git a/composer.lock b/composer.lock
index b2b7ed3..c768e94 100644
--- a/composer.lock
+++ b/composer.lock
@@ -1,10 +1,10 @@
 {
     "_readme": [
         "This file locks the dependencies of your project to a known state",
-        "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
+        "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "2891626665524c1259b4c522176d8296",
+    "content-hash": "0e3e5b63c84c3a395b6bb4f2a7478780",
     "packages": [
         {
             "name": "squizlabs/php_codesniffer",
@@ -366,6 +366,98 @@
             ],
             "time": "2019-02-06T07:57:58+00:00"
         },
+        {
+            "name": "symfony/twig-bridge",
+            "version": "v4.2.8",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/symfony/twig-bridge.git",
+                "reference": "4dcd115799163409c74ad54e9a3788211daa9f74"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/symfony/twig-bridge/zipball/4dcd115799163409c74ad54e9a3788211daa9f74",
+                "reference": "4dcd115799163409c74ad54e9a3788211daa9f74",
+                "shasum": ""
+            },
+            "require": {
+                "php": "^7.1.3",
+                "symfony/contracts": "^1.0.2",
+                "twig/twig": "^1.40|^2.9"
+            },
+            "conflict": {
+                "symfony/console": "<3.4",
+                "symfony/form": "<4.2.4",
+                "symfony/translation": "<4.2"
+            },
+            "require-dev": {
+                "symfony/asset": "~3.4|~4.0",
+                "symfony/console": "~3.4|~4.0",
+                "symfony/dependency-injection": "~3.4|~4.0",
+                "symfony/expression-language": "~3.4|~4.0",
+                "symfony/finder": "~3.4|~4.0",
+                "symfony/form": "^4.2.4",
+                "symfony/http-foundation": "~3.4|~4.0",
+                "symfony/http-kernel": "~3.4|~4.0",
+                "symfony/polyfill-intl-icu": "~1.0",
+                "symfony/routing": "~3.4|~4.0",
+                "symfony/security": "~3.4|~4.0",
+                "symfony/security-acl": "~2.8|~3.0",
+                "symfony/stopwatch": "~3.4|~4.0",
+                "symfony/templating": "~3.4|~4.0",
+                "symfony/translation": "~4.2",
+                "symfony/var-dumper": "~3.4|~4.0",
+                "symfony/web-link": "~3.4|~4.0",
+                "symfony/workflow": "~3.4|~4.0",
+                "symfony/yaml": "~3.4|~4.0"
+            },
+            "suggest": {
+                "symfony/asset": "For using the AssetExtension",
+                "symfony/expression-language": "For using the ExpressionExtension",
+                "symfony/finder": "",
+                "symfony/form": "For using the FormExtension",
+                "symfony/http-kernel": "For using the HttpKernelExtension",
+                "symfony/routing": "For using the RoutingExtension",
+                "symfony/security": "For using the SecurityExtension",
+                "symfony/stopwatch": "For using the StopwatchExtension",
+                "symfony/templating": "For using the TwigEngine",
+                "symfony/translation": "For using the TranslationExtension",
+                "symfony/var-dumper": "For using the DumpExtension",
+                "symfony/web-link": "For using the WebLinkExtension",
+                "symfony/yaml": "For using the YamlExtension"
+            },
+            "type": "symfony-bridge",
+            "extra": {
+                "branch-alias": {
+                    "dev-master": "4.2-dev"
+                }
+            },
+            "autoload": {
+                "psr-4": {
+                    "Symfony\\Bridge\\Twig\\": ""
+                },
+                "exclude-from-classmap": [
+                    "/Tests/"
+                ]
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Fabien Potencier",
+                    "email": "fabien@symfony.com"
+                },
+                {
+                    "name": "Symfony Community",
+                    "homepage": "https://symfony.com/contributors"
+                }
+            ],
+            "description": "Symfony Twig Bridge",
+            "homepage": "https://symfony.com",
+            "time": "2019-05-01T05:55:04+00:00"
+        },
         {
             "name": "twig/twig",
             "version": "v2.10.0",
@@ -1855,6 +1947,8 @@
     "stability-flags": [],
     "prefer-stable": false,
     "prefer-lowest": false,
-    "platform": [],
+    "platform": {
+        "php": ">=7.0"
+    },
     "platform-dev": []
 }