Skip to content

Commit e90c406

Browse files
VincentLangletVincentLanglet
committed
WIP
1 parent 5df5732 commit e90c406

File tree

6 files changed

+175
-49
lines changed

6 files changed

+175
-49
lines changed

‎TwigCS/Tests/AbstractSniffTest.php

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,18 @@ protected function checkGenericSniff(SniffInterface $sniff, array $expects): voi
4949
return;
5050
}
5151

52+
$fixedFile = __DIR__.'/Fixtures/'.$className.'.fixed.twig';
53+
if (file_exists($fixedFile)) {
54+
$fixer = new Fixer($ruleset, $tokenizer);
55+
$sniff->enableFixer($fixer);
56+
$fixer->fixFile($file);
57+
58+
$diff = $fixer->generateDiff($fixedFile);
59+
if ('' !== $diff) {
60+
self::fail($diff);
61+
}
62+
}
63+
5264
$messages = $report->getMessages();
5365
$messagePositions = [];
5466

@@ -60,23 +72,11 @@ protected function checkGenericSniff(SniffInterface $sniff, array $expects): voi
6072
if (null !== $line) {
6173
$errorMessage = sprintf('Line %s: %s', $line, $errorMessage);
6274
}
63-
$this->fail($errorMessage);
75+
self::fail($errorMessage);
6476
}
6577

6678
$messagePositions[] = [$message->getLine() => $message->getLinePosition()];
6779
}
68-
$this->assertEquals($expects, $messagePositions);
69-
70-
$fixedFile = __DIR__.'/Fixtures/'.$className.'.fixed.twig';
71-
if (file_exists($fixedFile)) {
72-
$fixer = new Fixer($ruleset, $tokenizer);
73-
$sniff->enableFixer($fixer);
74-
$fixer->fixFile($file);
75-
76-
$diff = $fixer->generateDiff($fixedFile);
77-
if ('' !== $diff) {
78-
$this->fail($diff);
79-
}
80-
}
80+
self::assertEquals($expects, $messagePositions);
8181
}
8282
}

‎TwigCS/Tests/Fixtures/OperatorSpacingTest.fixed.twig

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,6 @@ Untouch +-/*%==:
2929
{{ 1..10 }}
3030
{{ -5..-2 }}
3131
{{ [1, -2] }}
32+
{% if -2 == -3 or -2 == -3 %}{% endif %}
33+
{% if a - 2 == -4 %}{% endif %}
34+
{{ a in -2..-3 }}

‎TwigCS/Tests/Fixtures/OperatorSpacingTest.twig

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,6 @@ Untouch +-/*%==:
2929
{{ 1..10 }}
3030
{{ -5..-2 }}
3131
{{ [1, -2] }}
32+
{% if -2 == -3 or -2 == -3 %}{% endif %}
33+
{% if a - 2 == -4 %}{% endif %}
34+
{{ a in -2..-3 }}

‎TwigCS/src/Token/Token.php

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,15 +25,16 @@ class Token
2525
public const INTERPOLATION_END_TYPE = 11;
2626
public const ARROW_TYPE = 12;
2727
// New constants
28-
public const WHITESPACE_TYPE = 13;
29-
public const TAB_TYPE = 14;
30-
public const EOL_TYPE = 15;
31-
public const COMMENT_START_TYPE = 16;
32-
public const COMMENT_TEXT_TYPE = 17;
33-
public const COMMENT_WHITESPACE_TYPE = 18;
34-
public const COMMENT_TAB_TYPE = 19;
35-
public const COMMENT_EOL_TYPE = 20;
36-
public const COMMENT_END_TYPE = 21;
28+
public const BLOCK_TAG_TYPE = 13;
29+
public const WHITESPACE_TYPE = 14;
30+
public const TAB_TYPE = 15;
31+
public const EOL_TYPE = 16;
32+
public const COMMENT_START_TYPE = 17;
33+
public const COMMENT_TEXT_TYPE = 18;
34+
public const COMMENT_WHITESPACE_TYPE = 19;
35+
public const COMMENT_TAB_TYPE = 20;
36+
public const COMMENT_EOL_TYPE = 21;
37+
public const COMMENT_END_TYPE = 22;
3738

3839
public const EMPTY_TOKENS = [
3940
self::WHITESPACE_TYPE => self::WHITESPACE_TYPE,

‎TwigCS/src/Token/Tokenizer.php

Lines changed: 129 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ class Tokenizer
1616
private const STATE_DATA = 0;
1717
private const STATE_BLOCK = 1;
1818
private const STATE_VAR = 2;
19-
private const STATE_STRING = 3;
19+
private const STATE_DQ_STRING = 3;
2020
private const STATE_INTERPOLATION = 4;
2121
private const STATE_COMMENT = 5;
2222

@@ -103,11 +103,13 @@ public function __construct(Environment $env, array $options = [])
103103

104104
$tokenizerHelper = new TokenizerHelper($env, $this->options);
105105
$this->regexes = [
106-
'lex_block' => $tokenizerHelper->getBlockRegex(),
107-
'lex_comment' => $tokenizerHelper->getCommentRegex(),
108-
'lex_variable' => $tokenizerHelper->getVariableRegex(),
109-
'operator' => $tokenizerHelper->getOperatorRegex(),
110-
'lex_tokens_start' => $tokenizerHelper->getTokensStartRegex(),
106+
'lex_block' => $tokenizerHelper->getBlockRegex(),
107+
'lex_comment' => $tokenizerHelper->getCommentRegex(),
108+
'lex_variable' => $tokenizerHelper->getVariableRegex(),
109+
'operator' => $tokenizerHelper->getOperatorRegex(),
110+
'lex_tokens_start' => $tokenizerHelper->getTokensStartRegex(),
111+
'interpolation_start' => $tokenizerHelper->getInterpolationStartRegex(),
112+
'interpolation_end' => $tokenizerHelper->getInterpolationEndRegex(),
111113
];
112114
}
113115

@@ -150,6 +152,12 @@ public function tokenize(Source $source): array
150152
$this->lexData();
151153
}
152154
break;
155+
case self::STATE_DQ_STRING:
156+
$this->lexDqString();
157+
break;
158+
case self::STATE_INTERPOLATION:
159+
$this->lexInterpolation();
160+
break;
153161
default:
154162
throw new Exception('Unhandled state in tokenize', 1);
155163
}
@@ -277,35 +285,29 @@ protected function pushToken(int $type, string $value = null): void
277285
protected function lexExpression(): void
278286
{
279287
$currentToken = $this->code[$this->cursor];
288+
$nextToken = $this->code[$this->cursor + 1] ?? null;
280289

281290
if (preg_match('/\t/', $currentToken)) {
282291
$this->lexTab();
283292
} elseif (' ' === $currentToken) {
284293
$this->lexWhitespace();
285294
} elseif (PHP_EOL === $currentToken) {
286295
$this->lexEOL();
296+
} elseif ('=' === $currentToken && '>' === $nextToken) {
297+
$this->lexArrowFunction();
287298
} elseif (preg_match($this->regexes['operator'], $this->code, $match, 0, $this->cursor)) {
288299
$this->lexOperator($match[0]);
289300
} elseif (preg_match(self::REGEX_NAME, $this->code, $match, 0, $this->cursor)) {
290-
// names
291-
$this->pushToken(Token::NAME_TYPE, $match[0]);
292-
$this->moveCursor($match[0]);
301+
$this->lexName($match[0]);
293302
} elseif (preg_match(self::REGEX_NUMBER, $this->code, $match, 0, $this->cursor)) {
294-
// numbers
295-
$number = (float) $match[0]; // floats
296-
if (ctype_digit($match[0]) && $number <= PHP_INT_MAX) {
297-
$number = (int) $match[0]; // integers lower than the maximum
298-
}
299-
$this->pushToken(Token::NUMBER_TYPE, (string) $number);
300-
$this->moveCursor($match[0]);
303+
$this->lexNumber($match[0]);
301304
} elseif (false !== strpos(self::PUNCTUATION, $this->code[$this->cursor])) {
302305
$this->lexPunctuation();
303306
} elseif (preg_match(self::REGEX_STRING, $this->code, $match, 0, $this->cursor)) {
304-
// strings
305-
$this->pushToken(Token::STRING_TYPE, addcslashes(stripcslashes($match[0]), '\\'));
306-
$this->moveCursor($match[0]);
307+
$this->lexString($match[0]);
308+
} elseif (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, 0, $this->cursor)) {
309+
$this->lexStartDqString();
307310
} else {
308-
// unlexable
309311
throw new Exception(sprintf('Unexpected character "%s"', $currentToken));
310312
}
311313
}
@@ -368,6 +370,51 @@ protected function lexComment(): void
368370
}
369371
}
370372

373+
/**
374+
* @throws Exception
375+
*/
376+
protected function lexDqString(): void
377+
{
378+
if (preg_match($this->regexes['interpolation_start'], $this->code, $match, 0, $this->cursor)) {
379+
$this->lexStartInterpolation();
380+
} elseif (preg_match(self::REGEX_DQ_STRING_PART, $this->code, $match, 0, $this->cursor)
381+
&& strlen($match[0]) > 0
382+
) {
383+
$this->pushToken(Token::STRING_TYPE, stripcslashes($match[0]));
384+
$this->moveCursor($match[0]);
385+
} elseif (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, 0, $this->cursor)) {
386+
$bracket = array_pop($this->bracketsAndTernary);
387+
388+
if ('"' !== $this->code[$this->cursor]) {
389+
throw new Exception(sprintf('Unclosed "%s"', $bracket[0]));
390+
}
391+
392+
$this->popState();
393+
$this->moveCursor('"');
394+
} else {
395+
throw new Exception(sprintf('Unexpected character "%s"', $this->code[$this->cursor]));
396+
}
397+
}
398+
399+
/**
400+
* @throws Exception
401+
*/
402+
protected function lexInterpolation(): void
403+
{
404+
$bracket = end($this->bracketsAndTernary);
405+
406+
if ($this->options['interpolation'][0] === $bracket[0]
407+
&& preg_match($this->regexes['interpolation_end'], $this->code, $match, 0, $this->cursor)
408+
) {
409+
array_pop($this->bracketsAndTernary);
410+
$this->pushToken(Token::INTERPOLATION_END_TYPE);
411+
$this->moveCursor($match[0]);
412+
$this->popState();
413+
} else {
414+
$this->lexExpression();
415+
}
416+
}
417+
371418
/**
372419
* @param int $limit
373420
*/
@@ -387,10 +434,12 @@ protected function lexData(int $limit = 0): void
387434
$this->lexEOL();
388435
} elseif (preg_match('/\S+/', $this->code, $match, 0, $this->cursor)) {
389436
$value = $match[0];
437+
390438
// Stop if cursor reaches the next token start.
391439
if (0 !== $limit && $limit <= ($this->cursor + strlen($value))) {
392440
$value = substr($value, 0, $limit - $this->cursor);
393441
}
442+
394443
// Fixing token start among expressions and comments.
395444
$nbTokenStart = preg_match_all($this->regexes['lex_tokens_start'], $value, $matches);
396445
if ($nbTokenStart) {
@@ -431,6 +480,21 @@ protected function lexStart(): void
431480
$this->moveCursor($tokenStart['fullMatch']);
432481
}
433482

483+
protected function lexStartDqString(): void
484+
{
485+
$this->bracketsAndTernary[] = ['"', $this->line];
486+
$this->pushState(self::STATE_DQ_STRING);
487+
$this->moveCursor('"');
488+
}
489+
490+
protected function lexStartInterpolation(): void
491+
{
492+
$this->bracketsAndTernary[] = [$this->options['interpolation'][0], $this->line];
493+
$this->pushToken(Token::INTERPOLATION_START_TYPE);
494+
$this->pushState(self::STATE_INTERPOLATION);
495+
$this->moveCursor($this->options['interpolation'][0]);
496+
}
497+
434498
protected function lexTab(): void
435499
{
436500
$currentToken = $this->code[$this->cursor];
@@ -478,10 +542,16 @@ protected function lexEOL(): void
478542
$this->moveCursor($this->code[$this->cursor]);
479543
}
480544

545+
protected function lexArrowFunction(): void
546+
{
547+
$this->pushToken(Token::ARROW_TYPE, '=>');
548+
$this->moveCursor('=>');
549+
}
550+
481551
/**
482552
* @param string $operator
483553
*/
484-
protected function lexOperator($operator): void
554+
protected function lexOperator(string $operator): void
485555
{
486556
if ('?' === $operator) {
487557
$this->bracketsAndTernary[] = [$operator, $this->line];
@@ -492,6 +562,29 @@ protected function lexOperator($operator): void
492562
$this->moveCursor($operator);
493563
}
494564

565+
/**
566+
* @param string $name
567+
*/
568+
protected function lexName(string $name): void
569+
{
570+
$this->pushToken(Token::NAME_TYPE, $name);
571+
$this->moveCursor($name);
572+
}
573+
574+
/**
575+
* @param string $numberAsString
576+
*/
577+
protected function lexNumber(string $numberAsString): void
578+
{
579+
$number = (float) $numberAsString; // floats
580+
if (ctype_digit($numberAsString) && $number <= PHP_INT_MAX) {
581+
$number = (int) $numberAsString; // integers lower than the maximum
582+
}
583+
584+
$this->pushToken(Token::NUMBER_TYPE, (string) $number);
585+
$this->moveCursor($numberAsString);
586+
}
587+
495588
/**
496589
* @throws Exception
497590
*/
@@ -515,16 +608,26 @@ protected function lexPunctuation(): void
515608
throw new Exception(sprintf('Unexpected "%s"', $currentToken));
516609
}
517610

518-
$expect = array_pop($this->bracketsAndTernary)[0];
519-
if ('?' === $expect) {
520-
throw new Exception('Unclosed ternary');
611+
$bracket = array_pop($this->bracketsAndTernary);
612+
if ('?' === $bracket[0]) {
613+
// Because {{ foo ? 'yes' }} is the same as {{ foo ? 'yes' : '' }}
614+
$bracket = array_pop($this->bracketsAndTernary);
521615
}
522-
if (strtr($expect, '([{', ')]}') !== $currentToken) {
523-
throw new Exception(sprintf('Unclosed "%s"', $expect));
616+
if (strtr($bracket[0], '([{', ')]}') !== $currentToken) {
617+
throw new Exception(sprintf('Unclosed "%s"', $bracket[0]));
524618
}
525619
}
526620

527621
$this->pushToken(Token::PUNCTUATION_TYPE, $currentToken);
528622
$this->moveCursor($currentToken);
529623
}
624+
625+
/**
626+
* @param string $string
627+
*/
628+
protected function lexString(string $string): void
629+
{
630+
$this->pushToken(Token::STRING_TYPE, addcslashes(stripcslashes($string), '\\'));
631+
$this->moveCursor($string);
632+
}
530633
}

‎TwigCS/src/Token/TokenizerHelper.php

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,22 @@ public function getTokensStartRegex(): string
7979
.'/s';
8080
}
8181

82+
/**
83+
* @return string
84+
*/
85+
public function getInterpolationStartRegex(): string
86+
{
87+
return '{'.preg_quote($this->options['interpolation'][0], '#').'}A';
88+
}
89+
90+
/**
91+
* @return string
92+
*/
93+
public function getInterpolationEndRegex(): string
94+
{
95+
return '{'.preg_quote($this->options['interpolation'][1], '#').'}A';
96+
}
97+
8298
/**
8399
* @return string
84100
*/

0 commit comments

Comments
 (0)