From 16d1cead2903bde6e6354fb3713d1efd735eeb28 Mon Sep 17 00:00:00 2001 From: Vincent Langlet Date: Thu, 16 Jan 2020 02:40:05 +0100 Subject: [PATCH 1/3] :sparkles: Multiples tokenizer improvements to handle tag, interpolation, unary, ... --- .../Ruleset/Generic/OperatorSpacingSniff.php | 5 + TwigCS/src/Token/Token.php | 21 +- TwigCS/src/Token/Tokenizer.php | 243 ++++++++++++++---- TwigCS/src/Token/TokenizerHelper.php | 66 ++++- TwigCS/tests/AbstractSniffTest.php | 28 +- .../Fixtures/OperatorSpacingTest.fixed.twig | 4 + .../tests/Fixtures/OperatorSpacingTest.twig | 4 + .../Ruleset/Generic/OperatorSpacingTest.php | 2 + 8 files changed, 288 insertions(+), 85 deletions(-) diff --git a/TwigCS/src/Ruleset/Generic/OperatorSpacingSniff.php b/TwigCS/src/Ruleset/Generic/OperatorSpacingSniff.php index 963551c..d0dd5dc 100644 --- a/TwigCS/src/Ruleset/Generic/OperatorSpacingSniff.php +++ b/TwigCS/src/Ruleset/Generic/OperatorSpacingSniff.php @@ -81,6 +81,11 @@ private function isUnary(int $tokenPosition, array $tokens): bool return true; } + if ($this->isTokenMatching($previousToken, Token::BLOCK_TAG_TYPE)) { + // {% if -2 ... %} + return true; + } + return false; } } diff --git a/TwigCS/src/Token/Token.php b/TwigCS/src/Token/Token.php index a63118f..8cdf50e 100644 --- a/TwigCS/src/Token/Token.php +++ b/TwigCS/src/Token/Token.php @@ -25,15 +25,18 @@ class Token public const INTERPOLATION_END_TYPE = 11; public const ARROW_TYPE = 12; // New constants - public const WHITESPACE_TYPE = 13; - public const TAB_TYPE = 14; - public const EOL_TYPE = 15; - public const COMMENT_START_TYPE = 16; - public const COMMENT_TEXT_TYPE = 17; - public const COMMENT_WHITESPACE_TYPE = 18; - public const COMMENT_TAB_TYPE = 19; - public const COMMENT_EOL_TYPE = 20; - public const COMMENT_END_TYPE = 21; + public const DQ_STRING_START_TYPE = 13; + public const DQ_STRING_END_TYPE = 14; + public const BLOCK_TAG_TYPE = 15; + public const WHITESPACE_TYPE = 16; + public const TAB_TYPE = 17; + public const EOL_TYPE = 18; + public const COMMENT_START_TYPE = 19; + public const COMMENT_TEXT_TYPE = 20; + public const COMMENT_WHITESPACE_TYPE = 21; + public const COMMENT_TAB_TYPE = 22; + public const COMMENT_EOL_TYPE = 23; + public const COMMENT_END_TYPE = 24; public const EMPTY_TOKENS = [ self::WHITESPACE_TYPE => self::WHITESPACE_TYPE, diff --git a/TwigCS/src/Token/Tokenizer.php b/TwigCS/src/Token/Tokenizer.php index facc7e4..7d62761 100644 --- a/TwigCS/src/Token/Tokenizer.php +++ b/TwigCS/src/Token/Tokenizer.php @@ -16,7 +16,7 @@ class Tokenizer private const STATE_DATA = 0; private const STATE_BLOCK = 1; private const STATE_VAR = 2; - private const STATE_STRING = 3; + private const STATE_DQ_STRING = 3; private const STATE_INTERPOLATION = 4; private const STATE_COMMENT = 5; @@ -27,6 +27,9 @@ class Tokenizer private const REGEX_DQ_STRING_PART = '/[^#"\\\\]*(?:(?:\\\\.|#(?!\{))[^#"\\\\]*)*/As'; private const PUNCTUATION = '()[]{}:.,|'; + // TODO: Il faut surement changer les regex pour que "a#" et "a" soient traités de la même manière + // Car actuellement le second est DQ string (sans interpolation) alors que le premier est string + /** * @var array */ @@ -68,7 +71,7 @@ class Tokenizer protected $tokenPositions = []; /** - * @var int[] + * @var array[] */ protected $state = []; @@ -94,20 +97,23 @@ class Tokenizer public function __construct(Environment $env, array $options = []) { $this->options = array_merge([ - 'tag_comment' => ['{#', '#}'], - 'tag_block' => ['{%', '%}'], - 'tag_variable' => ['{{', '}}'], - 'whitespace_trim' => '-', - 'interpolation' => ['#{', '}'], + 'tag_comment' => ['{#', '#}'], + 'tag_block' => ['{%', '%}'], + 'tag_variable' => ['{{', '}}'], + 'whitespace_trim' => '-', + 'whitespace_line_trim' => '~', + 'interpolation' => ['#{', '}'], ], $options); $tokenizerHelper = new TokenizerHelper($env, $this->options); $this->regexes = [ - 'lex_block' => $tokenizerHelper->getBlockRegex(), - 'lex_comment' => $tokenizerHelper->getCommentRegex(), - 'lex_variable' => $tokenizerHelper->getVariableRegex(), - 'operator' => $tokenizerHelper->getOperatorRegex(), - 'lex_tokens_start' => $tokenizerHelper->getTokensStartRegex(), + 'lex_block' => $tokenizerHelper->getBlockRegex(), + 'lex_comment' => $tokenizerHelper->getCommentRegex(), + 'lex_variable' => $tokenizerHelper->getVariableRegex(), + 'operator' => $tokenizerHelper->getOperatorRegex(), + 'lex_tokens_start' => $tokenizerHelper->getTokensStartRegex(), + 'interpolation_start' => $tokenizerHelper->getInterpolationStartRegex(), + 'interpolation_end' => $tokenizerHelper->getInterpolationEndRegex(), ]; } @@ -150,6 +156,12 @@ public function tokenize(Source $source): array $this->lexData(); } break; + case self::STATE_DQ_STRING: + $this->lexDqString(); + break; + case self::STATE_INTERPOLATION: + $this->lexInterpolation(); + break; default: throw new Exception('Unhandled state in tokenize', 1); } @@ -186,15 +198,39 @@ protected function resetState(Source $source): void */ protected function getState(): int { - return count($this->state) > 0 ? $this->state[count($this->state) - 1] : self::STATE_DATA; + return count($this->state) > 0 ? $this->state[count($this->state) - 1][0] : self::STATE_DATA; + } + + /** + * @param int $state + * @param array $data + */ + protected function pushState(int $state, array $data = []): void + { + $this->state[] = [$state, $data]; + } + + /** + * @param string $name + * @param mixed $value + * + * @throws Exception + */ + protected function setStateParam($name, $value): void + { + if (0 === count($this->state)) { + throw new Exception('Cannot update state without a current state'); + } + + $this->state[count($this->state) - 1][1][$name] = $value; } /** - * @param int $state + * @return array */ - protected function pushState(int $state): void + protected function getStateParams(): array { - $this->state[] = $state; + return count($this->state) > 0 ? $this->state[count($this->state) - 1][1] : []; } /** @@ -203,7 +239,7 @@ protected function pushState(int $state): void protected function popState(): void { if (0 === count($this->state)) { - throw new Exception('Cannot pop state without a previous state'); + throw new Exception('Cannot pop state without a current state'); } array_pop($this->state); } @@ -277,6 +313,7 @@ protected function pushToken(int $type, string $value = null): void protected function lexExpression(): void { $currentToken = $this->code[$this->cursor]; + $nextToken = $this->code[$this->cursor + 1] ?? null; if (preg_match('/\t/', $currentToken)) { $this->lexTab(); @@ -284,28 +321,21 @@ protected function lexExpression(): void $this->lexWhitespace(); } elseif (PHP_EOL === $currentToken) { $this->lexEOL(); + } elseif ('=' === $currentToken && '>' === $nextToken) { + $this->lexArrowFunction(); } elseif (preg_match($this->regexes['operator'], $this->code, $match, 0, $this->cursor)) { $this->lexOperator($match[0]); } elseif (preg_match(self::REGEX_NAME, $this->code, $match, 0, $this->cursor)) { - // names - $this->pushToken(Token::NAME_TYPE, $match[0]); - $this->moveCursor($match[0]); + $this->lexName($match[0]); } elseif (preg_match(self::REGEX_NUMBER, $this->code, $match, 0, $this->cursor)) { - // numbers - $number = (float) $match[0]; // floats - if (ctype_digit($match[0]) && $number <= PHP_INT_MAX) { - $number = (int) $match[0]; // integers lower than the maximum - } - $this->pushToken(Token::NUMBER_TYPE, (string) $number); - $this->moveCursor($match[0]); + $this->lexNumber($match[0]); } elseif (false !== strpos(self::PUNCTUATION, $this->code[$this->cursor])) { $this->lexPunctuation(); } elseif (preg_match(self::REGEX_STRING, $this->code, $match, 0, $this->cursor)) { - // strings - $this->pushToken(Token::STRING_TYPE, addcslashes(stripcslashes($match[0]), '\\')); - $this->moveCursor($match[0]); + $this->lexString($match[0]); + } elseif (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, 0, $this->cursor)) { + $this->lexStartDqString(); } else { - // unlexable throw new Exception(sprintf('Unexpected character "%s"', $currentToken)); } } @@ -318,13 +348,13 @@ protected function lexBlock(): void $endRegex = $this->regexes['lex_block']; preg_match($endRegex, $this->code, $match, PREG_OFFSET_CAPTURE, $this->cursor); - if (count($this->bracketsAndTernary) > 0 || !isset($match[0])) { - $this->lexExpression(); - } else { + if (count($this->bracketsAndTernary) === 0 && isset($match[0])) { $this->pushToken(Token::BLOCK_END_TYPE, $match[0][0]); $this->moveCursor($match[0][0]); $this->moveCurrentPosition(); $this->popState(); + } else { + $this->lexExpression(); } } @@ -336,13 +366,13 @@ protected function lexVariable(): void $endRegex = $this->regexes['lex_variable']; preg_match($endRegex, $this->code, $match, PREG_OFFSET_CAPTURE, $this->cursor); - if (count($this->bracketsAndTernary) > 0 || !isset($match[0])) { - $this->lexExpression(); - } else { + if (count($this->bracketsAndTernary) === 0 && isset($match[0])) { $this->pushToken(Token::VAR_END_TYPE, $match[0][0]); $this->moveCursor($match[0][0]); $this->moveCurrentPosition(); $this->popState(); + } else { + $this->lexExpression(); } } @@ -368,6 +398,52 @@ protected function lexComment(): void } } + /** + * @throws Exception + */ + protected function lexDqString(): void + { + if (preg_match($this->regexes['interpolation_start'], $this->code, $match, 0, $this->cursor)) { + $this->lexStartInterpolation(); + } elseif (preg_match(self::REGEX_DQ_STRING_PART, $this->code, $match, 0, $this->cursor) + && strlen($match[0]) > 0 + ) { + $this->pushToken(Token::STRING_TYPE, stripcslashes($match[0])); + $this->moveCursor($match[0]); + } elseif (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, 0, $this->cursor)) { + $bracket = array_pop($this->bracketsAndTernary); + + if ('"' !== $this->code[$this->cursor]) { + throw new Exception(sprintf('Unclosed "%s"', $bracket[0])); + } + + $this->popState(); + $this->pushToken(Token::DQ_STRING_END_TYPE, $match[0]); + $this->moveCursor($match[0]); + } else { + throw new Exception(sprintf('Unexpected character "%s"', $this->code[$this->cursor])); + } + } + + /** + * @throws Exception + */ + protected function lexInterpolation(): void + { + $bracket = end($this->bracketsAndTernary); + + if ($this->options['interpolation'][0] === $bracket[0] + && preg_match($this->regexes['interpolation_end'], $this->code, $match, 0, $this->cursor) + ) { + array_pop($this->bracketsAndTernary); + $this->pushToken(Token::INTERPOLATION_END_TYPE, $match[0]); + $this->moveCursor($match[0]); + $this->popState(); + } else { + $this->lexExpression(); + } + } + /** * @param int $limit */ @@ -387,10 +463,12 @@ protected function lexData(int $limit = 0): void $this->lexEOL(); } elseif (preg_match('/\S+/', $this->code, $match, 0, $this->cursor)) { $value = $match[0]; + // Stop if cursor reaches the next token start. if (0 !== $limit && $limit <= ($this->cursor + strlen($value))) { $value = substr($value, 0, $limit - $this->cursor); } + // Fixing token start among expressions and comments. $nbTokenStart = preg_match_all($this->regexes['lex_tokens_start'], $value, $matches); if ($nbTokenStart) { @@ -431,6 +509,22 @@ protected function lexStart(): void $this->moveCursor($tokenStart['fullMatch']); } + protected function lexStartDqString(): void + { + $this->bracketsAndTernary[] = ['"', $this->line]; + $this->pushToken(Token::DQ_STRING_START_TYPE, '"'); + $this->pushState(self::STATE_DQ_STRING); + $this->moveCursor('"'); + } + + protected function lexStartInterpolation(): void + { + $this->bracketsAndTernary[] = [$this->options['interpolation'][0], $this->line]; + $this->pushToken(Token::INTERPOLATION_START_TYPE, '#{'); + $this->pushState(self::STATE_INTERPOLATION); + $this->moveCursor($this->options['interpolation'][0]); + } + protected function lexTab(): void { $currentToken = $this->code[$this->cursor]; @@ -478,20 +572,58 @@ protected function lexEOL(): void $this->moveCursor($this->code[$this->cursor]); } + protected function lexArrowFunction(): void + { + $this->pushToken(Token::ARROW_TYPE, '=>'); + $this->moveCursor('=>'); + } + /** * @param string $operator */ - protected function lexOperator($operator): void + protected function lexOperator(string $operator): void { if ('?' === $operator) { $this->bracketsAndTernary[] = [$operator, $this->line]; + } elseif (':' === $operator) { + array_pop($this->bracketsAndTernary); } - // operators $this->pushToken(Token::OPERATOR_TYPE, $operator); $this->moveCursor($operator); } + /** + * @param string $name + * + * @throws Exception + */ + protected function lexName(string $name): void + { + if (self::STATE_BLOCK === $this->getState() && !isset($this->getStateParams()['tag'])) { + $this->pushToken(Token::BLOCK_TAG_TYPE, $name); + $this->setStateParam('tag', $name); + } else { + $this->pushToken(Token::NAME_TYPE, $name); + } + + $this->moveCursor($name); + } + + /** + * @param string $numberAsString + */ + protected function lexNumber(string $numberAsString): void + { + $number = (float) $numberAsString; // floats + if (ctype_digit($numberAsString) && $number <= PHP_INT_MAX) { + $number = (int) $numberAsString; // integers lower than the maximum + } + + $this->pushToken(Token::NUMBER_TYPE, (string) $number); + $this->moveCursor($numberAsString); + } + /** * @throws Exception */ @@ -499,13 +631,14 @@ protected function lexPunctuation(): void { $currentToken = $this->code[$this->cursor]; - $lastBracket = end($this->bracketsAndTernary); - if (false !== $lastBracket && '?' === $lastBracket[0] && ':' === $currentToken) { - // This is a ternary instead - $this->lexOperator($currentToken); - array_pop($this->bracketsAndTernary); + if (':' === $currentToken) { + $lastBracket = end($this->bracketsAndTernary); + if (false !== $lastBracket && '?' === $lastBracket[0]) { + // This is a ternary instead + $this->lexOperator($currentToken); - return; + return; + } } if (false !== strpos('([{', $currentToken)) { @@ -515,16 +648,26 @@ protected function lexPunctuation(): void throw new Exception(sprintf('Unexpected "%s"', $currentToken)); } - $expect = array_pop($this->bracketsAndTernary)[0]; - if ('?' === $expect) { - throw new Exception('Unclosed ternary'); + $bracket = array_pop($this->bracketsAndTernary); + if ('?' === $bracket[0]) { + // Because {{ foo ? 'yes' }} is the same as {{ foo ? 'yes' : '' }} + $bracket = array_pop($this->bracketsAndTernary); } - if (strtr($expect, '([{', ')]}') !== $currentToken) { - throw new Exception(sprintf('Unclosed "%s"', $expect)); + if (strtr($bracket[0], '([{', ')]}') !== $currentToken) { + throw new Exception(sprintf('Unclosed "%s"', $bracket[0])); } } $this->pushToken(Token::PUNCTUATION_TYPE, $currentToken); $this->moveCursor($currentToken); } + + /** + * @param string $string + */ + protected function lexString(string $string): void + { + $this->pushToken(Token::STRING_TYPE, addcslashes(stripcslashes($string), '\\')); + $this->moveCursor($string); + } } diff --git a/TwigCS/src/Token/TokenizerHelper.php b/TwigCS/src/Token/TokenizerHelper.php index 934a4e7..52b9aaa 100644 --- a/TwigCS/src/Token/TokenizerHelper.php +++ b/TwigCS/src/Token/TokenizerHelper.php @@ -37,8 +37,13 @@ public function __construct(Environment $env, array $options = []) public function getBlockRegex(): string { return '/' - .'('.preg_quote($this->options['whitespace_trim']).')?' - .'('.preg_quote($this->options['tag_block'][1]).')' + .'(' + .preg_quote($this->options['whitespace_trim']) + .'|'.preg_quote($this->options['whitespace_line_trim']) + .')?' + .'(' + .preg_quote($this->options['tag_block'][1]) + .')' .'/A'; } @@ -48,9 +53,14 @@ public function getBlockRegex(): string public function getCommentRegex(): string { return '/' - .'('.preg_quote($this->options['whitespace_trim']).')?' - .'('.preg_quote($this->options['tag_comment'][1]).')' - .'/'; + .'(' + .preg_quote($this->options['whitespace_trim']) + .'|'.preg_quote($this->options['whitespace_line_trim']) + .')?' + .'(' + .preg_quote($this->options['tag_comment'][1]) + .')' + .'/'; // Should not be anchored } /** @@ -59,8 +69,13 @@ public function getCommentRegex(): string public function getVariableRegex(): string { return '/' - .'('.preg_quote($this->options['whitespace_trim']).')?' - .'('.preg_quote($this->options['tag_variable'][1]).')' + .'(' + .preg_quote($this->options['whitespace_trim']) + .'|'.preg_quote($this->options['whitespace_line_trim']) + .')?' + .'(' + .preg_quote($this->options['tag_variable'][1]) + .')' .'/A'; } @@ -71,12 +86,39 @@ public function getTokensStartRegex(): string { return '/' .'(' - .preg_quote($this->options['tag_variable'][0], '/') - .'|'.preg_quote($this->options['tag_block'][0], '/') - .'|'.preg_quote($this->options['tag_comment'][0], '/') + .preg_quote($this->options['tag_variable'][0]) + .'|'.preg_quote($this->options['tag_block'][0]) + .'|'.preg_quote($this->options['tag_comment'][0]) + .')' + .'(' + .preg_quote($this->options['whitespace_trim']) + .'|'.preg_quote($this->options['whitespace_line_trim']) + .')?' + .'/'; + } + + /** + * @return string + */ + public function getInterpolationStartRegex(): string + { + return '/' + .'(' + .preg_quote($this->options['interpolation'][0]) + .')' + .'/A'; + } + + /** + * @return string + */ + public function getInterpolationEndRegex(): string + { + return '/' + .'(' + .preg_quote($this->options['interpolation'][1]) .')' - .'('.preg_quote($this->options['whitespace_trim'], '/').')?' - .'/s'; + .'/A'; } /** diff --git a/TwigCS/tests/AbstractSniffTest.php b/TwigCS/tests/AbstractSniffTest.php index 80b0ccd..fe33376 100644 --- a/TwigCS/tests/AbstractSniffTest.php +++ b/TwigCS/tests/AbstractSniffTest.php @@ -49,6 +49,18 @@ protected function checkGenericSniff(SniffInterface $sniff, array $expects): voi return; } + $fixedFile = __DIR__.'/Fixtures/'.$className.'.fixed.twig'; + if (file_exists($fixedFile)) { + $fixer = new Fixer($ruleset, $tokenizer); + $sniff->enableFixer($fixer); + $fixer->fixFile($file); + + $diff = $fixer->generateDiff($fixedFile); + if ('' !== $diff) { + self::fail($diff); + } + } + $messages = $report->getMessages(); $messagePositions = []; @@ -60,23 +72,11 @@ protected function checkGenericSniff(SniffInterface $sniff, array $expects): voi if (null !== $line) { $errorMessage = sprintf('Line %s: %s', $line, $errorMessage); } - $this->fail($errorMessage); + self::fail($errorMessage); } $messagePositions[] = [$message->getLine() => $message->getLinePosition()]; } - $this->assertEquals($expects, $messagePositions); - - $fixedFile = __DIR__.'/Fixtures/'.$className.'.fixed.twig'; - if (file_exists($fixedFile)) { - $fixer = new Fixer($ruleset, $tokenizer); - $sniff->enableFixer($fixer); - $fixer->fixFile($file); - - $diff = $fixer->generateDiff($fixedFile); - if ('' !== $diff) { - $this->fail($diff); - } - } + self::assertEquals($expects, $messagePositions); } } diff --git a/TwigCS/tests/Fixtures/OperatorSpacingTest.fixed.twig b/TwigCS/tests/Fixtures/OperatorSpacingTest.fixed.twig index cf2e092..e99904f 100644 --- a/TwigCS/tests/Fixtures/OperatorSpacingTest.fixed.twig +++ b/TwigCS/tests/Fixtures/OperatorSpacingTest.fixed.twig @@ -29,3 +29,7 @@ Untouch +-/*%==: {{ 1..10 }} {{ -5..-2 }} {{ [1, -2] }} +{% if -2 == -3 or -2 == -3 %}{% endif %} +{% if a - 2 == -4 %}{% endif %} +{{ a in -2..-3 }} +{{ "a_#{1 + 1}" }} diff --git a/TwigCS/tests/Fixtures/OperatorSpacingTest.twig b/TwigCS/tests/Fixtures/OperatorSpacingTest.twig index 02c1078..bfc2fc9 100644 --- a/TwigCS/tests/Fixtures/OperatorSpacingTest.twig +++ b/TwigCS/tests/Fixtures/OperatorSpacingTest.twig @@ -29,3 +29,7 @@ Untouch +-/*%==: {{ 1..10 }} {{ -5..-2 }} {{ [1, -2] }} +{% if -2 == -3 or -2 == -3 %}{% endif %} +{% if a - 2 == -4 %}{% endif %} +{{ a in -2..-3 }} +{{ "a_#{1+1}" }} diff --git a/TwigCS/tests/Ruleset/Generic/OperatorSpacingTest.php b/TwigCS/tests/Ruleset/Generic/OperatorSpacingTest.php index a21a5b1..05a946b 100644 --- a/TwigCS/tests/Ruleset/Generic/OperatorSpacingTest.php +++ b/TwigCS/tests/Ruleset/Generic/OperatorSpacingTest.php @@ -51,6 +51,8 @@ public function testSniff(): void [21 => 5], [22 => 5], [22 => 5], + [35 => 10], + [35 => 10], ]); } } From 2da18fdd5e7c8294f4611c4c6c3b8fabdf519531 Mon Sep 17 00:00:00 2001 From: Vincent Langlet Date: Fri, 7 Feb 2020 01:53:13 +0100 Subject: [PATCH 2/3] :bug: Fix string with / --- TwigCS/src/Token/Tokenizer.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/TwigCS/src/Token/Tokenizer.php b/TwigCS/src/Token/Tokenizer.php index 7d62761..07d6f27 100644 --- a/TwigCS/src/Token/Tokenizer.php +++ b/TwigCS/src/Token/Tokenizer.php @@ -20,16 +20,16 @@ class Tokenizer private const STATE_INTERPOLATION = 4; private const STATE_COMMENT = 5; + private const SQ_STRING_PART = '[^\'\\\\]*(?:\\\\.[^\'\\\\]*)*'; + private const DQ_STRING_PART = '[^#"\\\\]*(?:(?:\\\\.|#(?!\{))[^#"\\\\]*)*'; + private const REGEX_NAME = '/[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*/A'; private const REGEX_NUMBER = '/[0-9]+(?:\.[0-9]+)?/A'; - private const REGEX_STRING = '/"([^#"\\\\]*(?:\\\\.[^#"\\\\]*)*)"|\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'/As'; + private const REGEX_STRING = '/"('.self::DQ_STRING_PART.')"|\'('.self::SQ_STRING_PART.')\'/As'; + private const REGEX_DQ_STRING_PART = '/'.self::DQ_STRING_PART.'/As'; private const REGEX_DQ_STRING_DELIM = '/"/A'; - private const REGEX_DQ_STRING_PART = '/[^#"\\\\]*(?:(?:\\\\.|#(?!\{))[^#"\\\\]*)*/As'; private const PUNCTUATION = '()[]{}:.,|'; - // TODO: Il faut surement changer les regex pour que "a#" et "a" soient traités de la même manière - // Car actuellement le second est DQ string (sans interpolation) alors que le premier est string - /** * @var array */ @@ -408,7 +408,7 @@ protected function lexDqString(): void } elseif (preg_match(self::REGEX_DQ_STRING_PART, $this->code, $match, 0, $this->cursor) && strlen($match[0]) > 0 ) { - $this->pushToken(Token::STRING_TYPE, stripcslashes($match[0])); + $this->pushToken(Token::STRING_TYPE, $match[0]); $this->moveCursor($match[0]); } elseif (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, 0, $this->cursor)) { $bracket = array_pop($this->bracketsAndTernary); @@ -667,7 +667,7 @@ protected function lexPunctuation(): void */ protected function lexString(string $string): void { - $this->pushToken(Token::STRING_TYPE, addcslashes(stripcslashes($string), '\\')); + $this->pushToken(Token::STRING_TYPE, $string); $this->moveCursor($string); } } From 77d46730ab3e124e804ce06f58bf6b7e1e227798 Mon Sep 17 00:00:00 2001 From: Vincent Langlet Date: Wed, 12 Feb 2020 01:04:12 +0100 Subject: [PATCH 3/3] :rotating_light: Add more tests --- SymfonyCustom/Helpers/AbstractHelper.php | 15 ++ SymfonyCustom/Helpers/FixerHelper.php | 6 +- SymfonyCustom/Helpers/SniffHelper.php | 6 +- TwigCS/src/Runner/Fixer.php | 133 ++++-------- TwigCS/src/Runner/Linter.php | 4 +- TwigCS/src/Sniff/AbstractSniff.php | 60 +++--- TwigCS/src/Sniff/AbstractSpacingSniff.php | 4 +- .../tests/Fixtures/OperatorSpacingTest.twig | 2 +- .../tests/{ => Ruleset}/AbstractSniffTest.php | 18 +- .../Generic/BlankEOF/BlankEOFTest.fixed.twig | 2 + .../Generic/{ => BlankEOF}/BlankEOFTest.php | 6 +- .../Generic/BlankEOF/BlankEOFTest.twig | 3 + .../DelimiterSpacingTest.fixed.twig | 20 ++ .../DelimiterSpacingTest.php | 6 +- .../DelimiterSpacingTest.twig | 20 ++ .../EmptyLines/EmptyLinesTest.fixed.twig | 3 + .../{ => EmptyLines}/EmptyLinesTest.php | 6 +- .../Generic/EmptyLines/EmptyLinesTest.twig | 4 + .../OperatorSpacingTest.fixed.twig | 35 ++++ .../OperatorSpacingTest.php | 7 +- .../OperatorSpacing/OperatorSpacingTest.twig | 35 ++++ TwigCS/tests/TestHelper.php | 73 +++++++ .../tests/Token/Tokenizer/TokenizerTest.php | 190 ++++++++++++++++++ .../tests/Token/Tokenizer/TokenizerTest1.twig | 1 + .../tests/Token/Tokenizer/TokenizerTest2.twig | 3 + .../tests/Token/Tokenizer/TokenizerTest3.twig | 3 + .../tests/Token/Tokenizer/TokenizerTest4.twig | 1 + 27 files changed, 499 insertions(+), 167 deletions(-) create mode 100644 SymfonyCustom/Helpers/AbstractHelper.php rename TwigCS/tests/{ => Ruleset}/AbstractSniffTest.php (77%) create mode 100644 TwigCS/tests/Ruleset/Generic/BlankEOF/BlankEOFTest.fixed.twig rename TwigCS/tests/Ruleset/Generic/{ => BlankEOF}/BlankEOFTest.php (62%) create mode 100644 TwigCS/tests/Ruleset/Generic/BlankEOF/BlankEOFTest.twig create mode 100644 TwigCS/tests/Ruleset/Generic/DelimiterSpacing/DelimiterSpacingTest.fixed.twig rename TwigCS/tests/Ruleset/Generic/{ => DelimiterSpacing}/DelimiterSpacingTest.php (68%) create mode 100644 TwigCS/tests/Ruleset/Generic/DelimiterSpacing/DelimiterSpacingTest.twig create mode 100644 TwigCS/tests/Ruleset/Generic/EmptyLines/EmptyLinesTest.fixed.twig rename TwigCS/tests/Ruleset/Generic/{ => EmptyLines}/EmptyLinesTest.php (62%) create mode 100644 TwigCS/tests/Ruleset/Generic/EmptyLines/EmptyLinesTest.twig create mode 100644 TwigCS/tests/Ruleset/Generic/OperatorSpacing/OperatorSpacingTest.fixed.twig rename TwigCS/tests/Ruleset/Generic/{ => OperatorSpacing}/OperatorSpacingTest.php (86%) create mode 100644 TwigCS/tests/Ruleset/Generic/OperatorSpacing/OperatorSpacingTest.twig create mode 100644 TwigCS/tests/TestHelper.php create mode 100644 TwigCS/tests/Token/Tokenizer/TokenizerTest.php create mode 100644 TwigCS/tests/Token/Tokenizer/TokenizerTest1.twig create mode 100644 TwigCS/tests/Token/Tokenizer/TokenizerTest2.twig create mode 100644 TwigCS/tests/Token/Tokenizer/TokenizerTest3.twig create mode 100644 TwigCS/tests/Token/Tokenizer/TokenizerTest4.twig diff --git a/SymfonyCustom/Helpers/AbstractHelper.php b/SymfonyCustom/Helpers/AbstractHelper.php new file mode 100644 index 0000000..6b97de8 --- /dev/null +++ b/SymfonyCustom/Helpers/AbstractHelper.php @@ -0,0 +1,15 @@ +getContents(); - - $tempName = tempnam(sys_get_temp_dir(), 'phpcs-fixer'); - $fixedFile = fopen($tempName, 'w'); - fwrite($fixedFile, $contents); - - // We must use something like shell_exec() because whitespace at the end - // of lines is critical to diff files. - $filename = escapeshellarg($filename); - $cmd = "diff -u -L$filename -LPHP_CodeSniffer $filename \"$tempName\""; - - $diff = shell_exec($cmd); - - fclose($fixedFile); - if (is_file($tempName) === true) { - unlink($tempName); - } - - $diffLines = null !== $diff ? explode(PHP_EOL, $diff) : []; - if (count($diffLines) === 1) { - // Seems to be required for cygwin. - $diffLines = explode("\n", $diff); - } - - $diff = []; - foreach ($diffLines as $line) { - if (isset($line[0]) === true) { - switch ($line[0]) { - case '-': - $diff[] = "\033[31m$line\033[0m"; - break; - case '+': - $diff[] = "\033[32m$line\033[0m"; - break; - default: - $diff[] = $line; - } - } - } - - $diff = implode(PHP_EOL, $diff); - - return $diff; - } - /** * @return string */ public function getContents(): string { - $contents = implode($this->tokens); - - return $contents; - } - - /** - * This function takes changesets into account so should be used - * instead of directly accessing the token array. - * - * @param int $tokenPosition - * - * @return string - */ - public function getTokenContent(int $tokenPosition): string - { - if ($this->inChangeset && isset($this->changeset[$tokenPosition])) { - return $this->changeset[$tokenPosition]; - } - - return $this->tokens[$tokenPosition]; + return implode($this->tokens); } /** @@ -367,24 +289,6 @@ public function replaceToken(int $tokenPosition, string $content): bool return true; } - /** - * @param int $tokenPosition - * - * @return bool - */ - public function revertToken(int $tokenPosition): bool - { - if (!isset($this->fixedTokens[$tokenPosition])) { - return false; - } - - $this->tokens[$tokenPosition] = $this->fixedTokens[$tokenPosition]; - unset($this->fixedTokens[$tokenPosition]); - $this->numFixes--; - - return true; - } - /** * @param int $tokenPosition * @@ -434,4 +338,39 @@ public function addContentBefore(int $tokenPosition, string $content): bool return $this->replaceToken($tokenPosition, $content.$current); } + + /** + * This function takes changesets into account so should be used + * instead of directly accessing the token array. + * + * @param int $tokenPosition + * + * @return string + */ + protected function getTokenContent(int $tokenPosition): string + { + if ($this->inChangeset && isset($this->changeset[$tokenPosition])) { + return $this->changeset[$tokenPosition]; + } + + return $this->tokens[$tokenPosition]; + } + + /** + * @param int $tokenPosition + * + * @return bool + */ + protected function revertToken(int $tokenPosition): bool + { + if (!isset($this->fixedTokens[$tokenPosition])) { + return false; + } + + $this->tokens[$tokenPosition] = $this->fixedTokens[$tokenPosition]; + unset($this->fixedTokens[$tokenPosition]); + $this->numFixes--; + + return true; + } } diff --git a/TwigCS/src/Runner/Linter.php b/TwigCS/src/Runner/Linter.php index 5a25470..31e8b25 100644 --- a/TwigCS/src/Runner/Linter.php +++ b/TwigCS/src/Runner/Linter.php @@ -86,7 +86,7 @@ public function run(array $files, Ruleset $ruleset, bool $fix = false): Report * * @throws Exception */ - public function fix(iterable $files, Ruleset $ruleset): void + protected function fix(iterable $files, Ruleset $ruleset): void { $fixer = new Fixer($ruleset, $this->tokenizer); @@ -113,7 +113,7 @@ public function fix(iterable $files, Ruleset $ruleset): void * * @return bool */ - public function processTemplate(string $file, Ruleset $ruleset, Report $report): bool + protected function processTemplate(string $file, Ruleset $ruleset, Report $report): bool { $twigSource = new Source(file_get_contents($file), $file); diff --git a/TwigCS/src/Sniff/AbstractSniff.php b/TwigCS/src/Sniff/AbstractSniff.php index fc63d28..569221e 100644 --- a/TwigCS/src/Sniff/AbstractSniff.php +++ b/TwigCS/src/Sniff/AbstractSniff.php @@ -47,6 +47,22 @@ public function disable(): void $this->fixer = null; } + /** + * @param array $stream + */ + public function processFile(array $stream): void + { + foreach ($stream as $index => $token) { + $this->process($index, $stream); + } + } + + /** + * @param int $tokenPosition + * @param Token[] $stream + */ + abstract protected function process(int $tokenPosition, array $stream): void; + /** * @param Token $token * @param int|array $type @@ -54,7 +70,7 @@ public function disable(): void * * @return bool */ - public function isTokenMatching(Token $token, $type, $value = []): bool + protected function isTokenMatching(Token $token, $type, $value = []): bool { if (!is_array($type)) { $type = [$type]; @@ -74,7 +90,7 @@ public function isTokenMatching(Token $token, $type, $value = []): bool * * @return int|false */ - public function findNext($type, array $tokens, int $start, bool $exclude = false) + protected function findNext($type, array $tokens, int $start, bool $exclude = false) { $i = 0; @@ -97,7 +113,7 @@ public function findNext($type, array $tokens, int $start, bool $exclude = false * * @return int|false */ - public function findPrevious($type, array $tokens, int $start, bool $exclude = false) + protected function findPrevious($type, array $tokens, int $start, bool $exclude = false) { $i = 0; @@ -118,7 +134,7 @@ public function findPrevious($type, array $tokens, int $start, bool $exclude = f * * @throws Exception */ - public function addWarning(string $message, Token $token): void + protected function addWarning(string $message, Token $token): void { $this->addMessage(Report::MESSAGE_TYPE_WARNING, $message, $token); } @@ -129,7 +145,7 @@ public function addWarning(string $message, Token $token): void * * @throws Exception */ - public function addError(string $message, Token $token): void + protected function addError(string $message, Token $token): void { $this->addMessage(Report::MESSAGE_TYPE_ERROR, $message, $token); } @@ -142,7 +158,7 @@ public function addError(string $message, Token $token): void * * @throws Exception */ - public function addFixableWarning(string $message, Token $token): bool + protected function addFixableWarning(string $message, Token $token): bool { return $this->addFixableMessage(Report::MESSAGE_TYPE_WARNING, $message, $token); } @@ -155,41 +171,11 @@ public function addFixableWarning(string $message, Token $token): bool * * @throws Exception */ - public function addFixableError(string $message, Token $token): bool + protected function addFixableError(string $message, Token $token): bool { return $this->addFixableMessage(Report::MESSAGE_TYPE_ERROR, $message, $token); } - /** - * @param Token $token - * - * @return string|null - */ - public function stringifyValue(Token $token): ?string - { - if ($token->getType() === Token::STRING_TYPE) { - return $token->getValue(); - } - - return '\''.$token->getValue().'\''; - } - - /** - * @param array $stream - */ - public function processFile(array $stream): void - { - foreach ($stream as $index => $token) { - $this->process($index, $stream); - } - } - - /** - * @param int $tokenPosition - * @param Token[] $stream - */ - abstract protected function process(int $tokenPosition, array $stream): void; - /** * @param int $messageType * @param string $message diff --git a/TwigCS/src/Sniff/AbstractSpacingSniff.php b/TwigCS/src/Sniff/AbstractSpacingSniff.php index 1e693d8..4e1e153 100644 --- a/TwigCS/src/Sniff/AbstractSpacingSniff.php +++ b/TwigCS/src/Sniff/AbstractSpacingSniff.php @@ -55,7 +55,7 @@ abstract protected function shouldHaveSpaceBefore(int $tokenPosition, array $tok * * @throws Exception */ - protected function checkSpaceAfter(int $tokenPosition, array $tokens, int $expected): void + private function checkSpaceAfter(int $tokenPosition, array $tokens, int $expected): void { $token = $tokens[$tokenPosition]; @@ -94,7 +94,7 @@ protected function checkSpaceAfter(int $tokenPosition, array $tokens, int $expec * * @throws Exception */ - protected function checkSpaceBefore(int $tokenPosition, array $tokens, int $expected): void + private function checkSpaceBefore(int $tokenPosition, array $tokens, int $expected): void { $token = $tokens[$tokenPosition]; diff --git a/TwigCS/tests/Fixtures/OperatorSpacingTest.twig b/TwigCS/tests/Fixtures/OperatorSpacingTest.twig index bfc2fc9..6827dbb 100644 --- a/TwigCS/tests/Fixtures/OperatorSpacingTest.twig +++ b/TwigCS/tests/Fixtures/OperatorSpacingTest.twig @@ -21,7 +21,7 @@ Untouch +-/*%==: {{ 1?:2 }} {{ 1??2 }} -{{ 1 + -2 }} +{{ 1 +-2 }} {{ 1 + (-2 + 3) }} {{ a[-2] }} {{ { 'foo': -2 } }} diff --git a/TwigCS/tests/AbstractSniffTest.php b/TwigCS/tests/Ruleset/AbstractSniffTest.php similarity index 77% rename from TwigCS/tests/AbstractSniffTest.php rename to TwigCS/tests/Ruleset/AbstractSniffTest.php index fe33376..132b240 100644 --- a/TwigCS/tests/AbstractSniffTest.php +++ b/TwigCS/tests/Ruleset/AbstractSniffTest.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace TwigCS\Tests; +namespace TwigCS\Tests\Ruleset; use Exception; use PHPUnit\Framework\TestCase; @@ -13,6 +13,7 @@ use TwigCS\Runner\Fixer; use TwigCS\Runner\Linter; use TwigCS\Sniff\SniffInterface; +use TwigCS\Tests\TestHelper; use TwigCS\Token\Tokenizer; /** @@ -21,7 +22,7 @@ abstract class AbstractSniffTest extends TestCase { /** - * Should call $this->checkGenericSniff(new Sniff(), [...]); + * Should call $this->checkSniff(new Sniff(), [...]); */ abstract public function testSniff(): void; @@ -29,7 +30,7 @@ abstract public function testSniff(): void; * @param SniffInterface $sniff * @param array $expects */ - protected function checkGenericSniff(SniffInterface $sniff, array $expects): void + protected function checkSniff(SniffInterface $sniff, array $expects): void { $env = new StubbedEnvironment(); $tokenizer = new Tokenizer($env); @@ -39,23 +40,24 @@ protected function checkGenericSniff(SniffInterface $sniff, array $expects): voi try { $class = new ReflectionClass(get_called_class()); $className = $class->getShortName(); - $file = __DIR__.'/Fixtures/'.$className.'.twig'; + $directory = dirname($class->getFileName()); + $file = "$directory/$className.twig"; $ruleset->addSniff($sniff); $report = $linter->run([$file], $ruleset); } catch (Exception $e) { - $this->fail($e->getMessage()); + self::fail($e->getMessage()); return; } - $fixedFile = __DIR__.'/Fixtures/'.$className.'.fixed.twig'; + $fixedFile = "$directory/$className.fixed.twig"; if (file_exists($fixedFile)) { $fixer = new Fixer($ruleset, $tokenizer); $sniff->enableFixer($fixer); $fixer->fixFile($file); - $diff = $fixer->generateDiff($fixedFile); + $diff = TestHelper::generateDiff($fixer->getContents(), $fixedFile); if ('' !== $diff) { self::fail($diff); } @@ -77,6 +79,6 @@ protected function checkGenericSniff(SniffInterface $sniff, array $expects): voi $messagePositions[] = [$message->getLine() => $message->getLinePosition()]; } - self::assertEquals($expects, $messagePositions); + self::assertSame($expects, $messagePositions); } } diff --git a/TwigCS/tests/Ruleset/Generic/BlankEOF/BlankEOFTest.fixed.twig b/TwigCS/tests/Ruleset/Generic/BlankEOF/BlankEOFTest.fixed.twig new file mode 100644 index 0000000..1936aaf --- /dev/null +++ b/TwigCS/tests/Ruleset/Generic/BlankEOF/BlankEOFTest.fixed.twig @@ -0,0 +1,2 @@ +
+
diff --git a/TwigCS/tests/Ruleset/Generic/BlankEOFTest.php b/TwigCS/tests/Ruleset/Generic/BlankEOF/BlankEOFTest.php similarity index 62% rename from TwigCS/tests/Ruleset/Generic/BlankEOFTest.php rename to TwigCS/tests/Ruleset/Generic/BlankEOF/BlankEOFTest.php index a6fd61c..1be86f7 100644 --- a/TwigCS/tests/Ruleset/Generic/BlankEOFTest.php +++ b/TwigCS/tests/Ruleset/Generic/BlankEOF/BlankEOFTest.php @@ -2,10 +2,10 @@ declare(strict_types=1); -namespace TwigCS\Tests\Ruleset\Generic; +namespace TwigCS\Tests\Ruleset\Generic\BlankEOF; use TwigCS\Ruleset\Generic\BlankEOFSniff; -use TwigCS\Tests\AbstractSniffTest; +use TwigCS\Tests\Ruleset\AbstractSniffTest; /** * Class BlankEOFTest @@ -14,7 +14,7 @@ class BlankEOFTest extends AbstractSniffTest { public function testSniff(): void { - $this->checkGenericSniff(new BlankEOFSniff(), [ + $this->checkSniff(new BlankEOFSniff(), [ [4 => 1], ]); } diff --git a/TwigCS/tests/Ruleset/Generic/BlankEOF/BlankEOFTest.twig b/TwigCS/tests/Ruleset/Generic/BlankEOF/BlankEOFTest.twig new file mode 100644 index 0000000..99dfd42 --- /dev/null +++ b/TwigCS/tests/Ruleset/Generic/BlankEOF/BlankEOFTest.twig @@ -0,0 +1,3 @@ +
+
+ diff --git a/TwigCS/tests/Ruleset/Generic/DelimiterSpacing/DelimiterSpacingTest.fixed.twig b/TwigCS/tests/Ruleset/Generic/DelimiterSpacing/DelimiterSpacingTest.fixed.twig new file mode 100644 index 0000000..692a539 --- /dev/null +++ b/TwigCS/tests/Ruleset/Generic/DelimiterSpacing/DelimiterSpacingTest.fixed.twig @@ -0,0 +1,20 @@ +{{ foo }} +{# comment #} +{% if foo %}{% endif %} + +{{- foo -}} +{#- comment -#} +{%- if foo -%}{%- endif -%} + +{{ + shouldNotCareAboutNewLine +}} +{%- if foo -%}{%- endif -%} + +{{ foo({'bar': {'baz': 'shouldNotCareAboutDoubleHashes'}}) }} + +
+ {{ + shouldNotCareAboutNewLine + }} +
diff --git a/TwigCS/tests/Ruleset/Generic/DelimiterSpacingTest.php b/TwigCS/tests/Ruleset/Generic/DelimiterSpacing/DelimiterSpacingTest.php similarity index 68% rename from TwigCS/tests/Ruleset/Generic/DelimiterSpacingTest.php rename to TwigCS/tests/Ruleset/Generic/DelimiterSpacing/DelimiterSpacingTest.php index 24b8c54..e11bd7d 100644 --- a/TwigCS/tests/Ruleset/Generic/DelimiterSpacingTest.php +++ b/TwigCS/tests/Ruleset/Generic/DelimiterSpacing/DelimiterSpacingTest.php @@ -2,10 +2,10 @@ declare(strict_types=1); -namespace TwigCS\Tests\Ruleset\Generic; +namespace TwigCS\Tests\Ruleset\Generic\DelimiterSpacing; use TwigCS\Ruleset\Generic\DelimiterSpacingSniff; -use TwigCS\Tests\AbstractSniffTest; +use TwigCS\Tests\Ruleset\AbstractSniffTest; /** * Class DelimiterSpacingTest @@ -14,7 +14,7 @@ class DelimiterSpacingTest extends AbstractSniffTest { public function testSniff(): void { - $this->checkGenericSniff(new DelimiterSpacingSniff(), [ + $this->checkSniff(new DelimiterSpacingSniff(), [ [12 => 1], [12 => 12], [12 => 15], diff --git a/TwigCS/tests/Ruleset/Generic/DelimiterSpacing/DelimiterSpacingTest.twig b/TwigCS/tests/Ruleset/Generic/DelimiterSpacing/DelimiterSpacingTest.twig new file mode 100644 index 0000000..0175972 --- /dev/null +++ b/TwigCS/tests/Ruleset/Generic/DelimiterSpacing/DelimiterSpacingTest.twig @@ -0,0 +1,20 @@ +{{ foo }} +{# comment #} +{% if foo %}{% endif %} + +{{- foo -}} +{#- comment -#} +{%- if foo -%}{%- endif -%} + +{{ + shouldNotCareAboutNewLine +}} +{%-if foo -%}{%- endif-%} + +{{ foo({'bar': {'baz': 'shouldNotCareAboutDoubleHashes'}}) }} + +
+ {{ + shouldNotCareAboutNewLine + }} +
diff --git a/TwigCS/tests/Ruleset/Generic/EmptyLines/EmptyLinesTest.fixed.twig b/TwigCS/tests/Ruleset/Generic/EmptyLines/EmptyLinesTest.fixed.twig new file mode 100644 index 0000000..70c6156 --- /dev/null +++ b/TwigCS/tests/Ruleset/Generic/EmptyLines/EmptyLinesTest.fixed.twig @@ -0,0 +1,3 @@ +
+ +
diff --git a/TwigCS/tests/Ruleset/Generic/EmptyLinesTest.php b/TwigCS/tests/Ruleset/Generic/EmptyLines/EmptyLinesTest.php similarity index 62% rename from TwigCS/tests/Ruleset/Generic/EmptyLinesTest.php rename to TwigCS/tests/Ruleset/Generic/EmptyLines/EmptyLinesTest.php index 49d0e8b..95203bd 100644 --- a/TwigCS/tests/Ruleset/Generic/EmptyLinesTest.php +++ b/TwigCS/tests/Ruleset/Generic/EmptyLines/EmptyLinesTest.php @@ -2,10 +2,10 @@ declare(strict_types=1); -namespace TwigCS\Tests\Ruleset\Generic; +namespace TwigCS\Tests\Ruleset\Generic\EmptyLines; use TwigCS\Ruleset\Generic\EmptyLinesSniff; -use TwigCS\Tests\AbstractSniffTest; +use TwigCS\Tests\Ruleset\AbstractSniffTest; /** * Class EmptyLinesTest @@ -14,7 +14,7 @@ class EmptyLinesTest extends AbstractSniffTest { public function testSniff(): void { - $this->checkGenericSniff(new EmptyLinesSniff(), [ + $this->checkSniff(new EmptyLinesSniff(), [ [3 => 1], ]); } diff --git a/TwigCS/tests/Ruleset/Generic/EmptyLines/EmptyLinesTest.twig b/TwigCS/tests/Ruleset/Generic/EmptyLines/EmptyLinesTest.twig new file mode 100644 index 0000000..018bc05 --- /dev/null +++ b/TwigCS/tests/Ruleset/Generic/EmptyLines/EmptyLinesTest.twig @@ -0,0 +1,4 @@ +
+ + +
diff --git a/TwigCS/tests/Ruleset/Generic/OperatorSpacing/OperatorSpacingTest.fixed.twig b/TwigCS/tests/Ruleset/Generic/OperatorSpacing/OperatorSpacingTest.fixed.twig new file mode 100644 index 0000000..e99904f --- /dev/null +++ b/TwigCS/tests/Ruleset/Generic/OperatorSpacing/OperatorSpacingTest.fixed.twig @@ -0,0 +1,35 @@ +{{ 1 + 2 }} +{{ 1 - 2 }} +{{ 1 / 2 }} +{{ 1 * 2 }} +{{ 1 % 2 }} +{{ 1 // 2 }} +{{ 1 ** 2 }} +{{ foo ~ bar }} +{{ true ? true : false }} +{{ 1 == 2 }} +{{ not true }} +{{ false and true }} +{{ false or true }} +{{ a in array }} +{{ a is array }} + +Untouch +-/*%==: + +{{ [1, 2, 3] }} +{{ {'foo': 'bar'} }} +{{ 1 ?: 2 }} +{{ 1 ?? 2 }} + +{{ 1 + -2 }} +{{ 1 + (-2 + 3) }} +{{ a[-2] }} +{{ { 'foo': -2 } }} +{{ foo.name }} +{{ 1..10 }} +{{ -5..-2 }} +{{ [1, -2] }} +{% if -2 == -3 or -2 == -3 %}{% endif %} +{% if a - 2 == -4 %}{% endif %} +{{ a in -2..-3 }} +{{ "a_#{1 + 1}" }} diff --git a/TwigCS/tests/Ruleset/Generic/OperatorSpacingTest.php b/TwigCS/tests/Ruleset/Generic/OperatorSpacing/OperatorSpacingTest.php similarity index 86% rename from TwigCS/tests/Ruleset/Generic/OperatorSpacingTest.php rename to TwigCS/tests/Ruleset/Generic/OperatorSpacing/OperatorSpacingTest.php index 05a946b..28c2889 100644 --- a/TwigCS/tests/Ruleset/Generic/OperatorSpacingTest.php +++ b/TwigCS/tests/Ruleset/Generic/OperatorSpacing/OperatorSpacingTest.php @@ -2,10 +2,10 @@ declare(strict_types=1); -namespace TwigCS\Tests\Ruleset\Generic; +namespace TwigCS\Tests\Ruleset\Generic\OperatorSpacing; use TwigCS\Ruleset\Generic\OperatorSpacingSniff; -use TwigCS\Tests\AbstractSniffTest; +use TwigCS\Tests\Ruleset\AbstractSniffTest; /** * Class OperatorSpacingTest @@ -14,7 +14,7 @@ class OperatorSpacingTest extends AbstractSniffTest { public function testSniff(): void { - $this->checkGenericSniff(new OperatorSpacingSniff(), [ + $this->checkSniff(new OperatorSpacingSniff(), [ [1 => 4], [1 => 4], [2 => 5], @@ -51,6 +51,7 @@ public function testSniff(): void [21 => 5], [22 => 5], [22 => 5], + [24 => 6], [35 => 10], [35 => 10], ]); diff --git a/TwigCS/tests/Ruleset/Generic/OperatorSpacing/OperatorSpacingTest.twig b/TwigCS/tests/Ruleset/Generic/OperatorSpacing/OperatorSpacingTest.twig new file mode 100644 index 0000000..6827dbb --- /dev/null +++ b/TwigCS/tests/Ruleset/Generic/OperatorSpacing/OperatorSpacingTest.twig @@ -0,0 +1,35 @@ +{{ 1+2 }} +{{ 1-2 }} +{{ 1/2 }} +{{ 1*2 }} +{{ 1%2 }} +{{ 1//2 }} +{{ 1**2 }} +{{ foo~bar }} +{{ true ? true : false }} +{{ 1==2 }} +{{ not true }} +{{ false and true }} +{{ false or true }} +{{ a in array }} +{{ a is array }} + +Untouch +-/*%==: + +{{ [1, 2, 3] }} +{{ {'foo': 'bar'} }} +{{ 1?:2 }} +{{ 1??2 }} + +{{ 1 +-2 }} +{{ 1 + (-2 + 3) }} +{{ a[-2] }} +{{ { 'foo': -2 } }} +{{ foo.name }} +{{ 1..10 }} +{{ -5..-2 }} +{{ [1, -2] }} +{% if -2 == -3 or -2 == -3 %}{% endif %} +{% if a - 2 == -4 %}{% endif %} +{{ a in -2..-3 }} +{{ "a_#{1+1}" }} diff --git a/TwigCS/tests/TestHelper.php b/TwigCS/tests/TestHelper.php new file mode 100644 index 0000000..75017cb --- /dev/null +++ b/TwigCS/tests/TestHelper.php @@ -0,0 +1,73 @@ +tokenize($source); + + $tokenValues = array_map(function (Token $token) { + return $token->getValue(); + }, $tokens); + + $diff = TestHelper::generateDiff(implode($tokenValues), $filePath); + if ('' !== $diff) { + self::fail($diff); + } + + $tokenTypes = array_map(function (Token $token) { + return $token->getType(); + }, $tokens); + self::assertSame($expectedTokenTypes, $tokenTypes); + } + + /** + * @return array + */ + public function tokenizeDataProvider(): array + { + return [ + [ + __DIR__.'/TokenizerTest1.twig', + [ + 0 => Token::TEXT_TYPE, + 1 => Token::EOL_TYPE, + 2 => Token::EOF_TYPE, + ], + ], + [ + __DIR__.'/TokenizerTest2.twig', + [ + 0 => Token::VAR_START_TYPE, + 1 => Token::WHITESPACE_TYPE, + 2 => Token::NAME_TYPE, + 3 => Token::WHITESPACE_TYPE, + 4 => Token::VAR_END_TYPE, + 5 => Token::EOL_TYPE, + 6 => Token::COMMENT_START_TYPE, + 7 => Token::COMMENT_WHITESPACE_TYPE, + 8 => Token::COMMENT_TEXT_TYPE, + 9 => Token::COMMENT_WHITESPACE_TYPE, + 10 => Token::COMMENT_END_TYPE, + 11 => Token::EOL_TYPE, + 12 => Token::BLOCK_START_TYPE, + 13 => Token::WHITESPACE_TYPE, + 14 => Token::BLOCK_TAG_TYPE, + 15 => Token::WHITESPACE_TYPE, + 16 => Token::NAME_TYPE, + 17 => Token::WHITESPACE_TYPE, + 18 => Token::BLOCK_END_TYPE, + 19 => Token::BLOCK_START_TYPE, + 20 => Token::WHITESPACE_TYPE, + 21 => Token::BLOCK_TAG_TYPE, + 22 => Token::WHITESPACE_TYPE, + 23 => Token::BLOCK_END_TYPE, + 24 => Token::EOL_TYPE, + 25 => Token::EOF_TYPE, + ], + ], + [ + __DIR__.'/TokenizerTest3.twig', + [ + 0 => Token::VAR_START_TYPE, + 1 => Token::WHITESPACE_TYPE, + 2 => Token::NUMBER_TYPE, + 3 => Token::OPERATOR_TYPE, + 4 => Token::NUMBER_TYPE, + 5 => Token::OPERATOR_TYPE, + 6 => Token::NUMBER_TYPE, + 7 => Token::OPERATOR_TYPE, + 8 => Token::NUMBER_TYPE, + 9 => Token::OPERATOR_TYPE, + 10 => Token::NUMBER_TYPE, + 11 => Token::WHITESPACE_TYPE, + 12 => Token::VAR_END_TYPE, + 13 => Token::EOL_TYPE, + 14 => Token::VAR_START_TYPE, + 15 => Token::WHITESPACE_TYPE, + 16 => Token::NAME_TYPE, + 17 => Token::WHITESPACE_TYPE, + 18 => Token::OPERATOR_TYPE, + 19 => Token::WHITESPACE_TYPE, + 20 => Token::STRING_TYPE, + 21 => Token::WHITESPACE_TYPE, + 22 => Token::OPERATOR_TYPE, + 23 => Token::WHITESPACE_TYPE, + 24 => Token::STRING_TYPE, + 25 => Token::WHITESPACE_TYPE, + 26 => Token::VAR_END_TYPE, + 27 => Token::EOL_TYPE, + 28 => Token::VAR_START_TYPE, + 29 => Token::WHITESPACE_TYPE, + 30 => Token::NAME_TYPE, + 31 => Token::WHITESPACE_TYPE, + 32 => Token::OPERATOR_TYPE, + 33 => Token::WHITESPACE_TYPE, + 34 => Token::PUNCTUATION_TYPE, + 35 => Token::WHITESPACE_TYPE, + 36 => Token::NAME_TYPE, + 37 => Token::PUNCTUATION_TYPE, + 38 => Token::PUNCTUATION_TYPE, + 39 => Token::NUMBER_TYPE, + 40 => Token::PUNCTUATION_TYPE, + 41 => Token::WHITESPACE_TYPE, + 42 => Token::NUMBER_TYPE, + 43 => Token::PUNCTUATION_TYPE, + 44 => Token::WHITESPACE_TYPE, + 45 => Token::PUNCTUATION_TYPE, + 46 => Token::WHITESPACE_TYPE, + 47 => Token::VAR_END_TYPE, + 48 => Token::EOL_TYPE, + 49 => Token::EOF_TYPE, + ], + ], + [ + __DIR__.'/TokenizerTest4.twig', + [ + 0 => Token::VAR_START_TYPE, + 1 => Token::WHITESPACE_TYPE, + 2 => Token::NAME_TYPE, + 3 => Token::PUNCTUATION_TYPE, + 4 => Token::NAME_TYPE, + 5 => Token::PUNCTUATION_TYPE, + 6 => Token::NAME_TYPE, + 7 => Token::WHITESPACE_TYPE, + 8 => Token::ARROW_TYPE, + 9 => Token::WHITESPACE_TYPE, + 10 => Token::DQ_STRING_START_TYPE, + 11 => Token::INTERPOLATION_START_TYPE, + 12 => Token::NAME_TYPE, + 13 => Token::PUNCTUATION_TYPE, + 14 => Token::NAME_TYPE, + 15 => Token::INTERPOLATION_END_TYPE, + 16 => Token::STRING_TYPE, + 17 => Token::INTERPOLATION_START_TYPE, + 18 => Token::NAME_TYPE, + 19 => Token::PUNCTUATION_TYPE, + 20 => Token::NAME_TYPE, + 21 => Token::INTERPOLATION_END_TYPE, + 22 => Token::DQ_STRING_END_TYPE, + 23 => Token::PUNCTUATION_TYPE, + 24 => Token::PUNCTUATION_TYPE, + 25 => Token::NAME_TYPE, + 26 => Token::PUNCTUATION_TYPE, + 27 => Token::STRING_TYPE, + 28 => Token::PUNCTUATION_TYPE, + 29 => Token::WHITESPACE_TYPE, + 30 => Token::VAR_END_TYPE, + 31 => Token::EOL_TYPE, + 32 => Token::EOF_TYPE, + ], + ], + ]; + } +} diff --git a/TwigCS/tests/Token/Tokenizer/TokenizerTest1.twig b/TwigCS/tests/Token/Tokenizer/TokenizerTest1.twig new file mode 100644 index 0000000..179f993 --- /dev/null +++ b/TwigCS/tests/Token/Tokenizer/TokenizerTest1.twig @@ -0,0 +1 @@ +
test
diff --git a/TwigCS/tests/Token/Tokenizer/TokenizerTest2.twig b/TwigCS/tests/Token/Tokenizer/TokenizerTest2.twig new file mode 100644 index 0000000..42e16f6 --- /dev/null +++ b/TwigCS/tests/Token/Tokenizer/TokenizerTest2.twig @@ -0,0 +1,3 @@ +{{ foo }} +{# comment #} +{% block try %}{% endblock %} diff --git a/TwigCS/tests/Token/Tokenizer/TokenizerTest3.twig b/TwigCS/tests/Token/Tokenizer/TokenizerTest3.twig new file mode 100644 index 0000000..faf2023 --- /dev/null +++ b/TwigCS/tests/Token/Tokenizer/TokenizerTest3.twig @@ -0,0 +1,3 @@ +{{ 1+2-3*4%5 }} +{{ bar ? 'string' : "string#" }} +{{ baz ?: { a:[1, 2] } }} diff --git a/TwigCS/tests/Token/Tokenizer/TokenizerTest4.twig b/TwigCS/tests/Token/Tokenizer/TokenizerTest4.twig new file mode 100644 index 0000000..339359d --- /dev/null +++ b/TwigCS/tests/Token/Tokenizer/TokenizerTest4.twig @@ -0,0 +1 @@ +{{ people|map(p => "#{p.first} #{p.last}")|join(', ') }}