Skip to content

Commit 98051e9

Browse files
fabpotfabpot
committed
feature #19604 [Routing] Add seamless support for unicode requirements (nicolas-grekas)
This PR was merged into the 3.2-dev branch. Discussion ---------- [Routing] Add seamless support for unicode requirements | Q | A | ------------- | --- | Branch? | master | Bug fix? | no | New feature? | yes | BC breaks? | no | Deprecations? | yes | Tests pass? | yes | Fixed tickets | #3629, #5236, #19562 | License | MIT | Doc PR | symfony/symfony-docs#6890 This PR adds unicode support to route matching and generation by automatically adding the `u` modifier to regexps that use either unicode characters or unicode enabled character classes (e.g. `\p...` `\x{...}` `\X`). As a side note, if one wants to match a single unicode character (vs a single byte), one should use `\PM` or `\X` instead of `.` *or* set the `unicode` parameter to true. Commits ------- a829d34 [Routing] Add seamless support for unicode requirements
2 parents df8cf70 + a829d34 commit 98051e9

File tree

8 files changed

+177
-13
lines changed

8 files changed

+177
-13
lines changed

‎src/Symfony/Component/Routing/CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ CHANGELOG
55
-----
66

77
* Added support for `bool`, `int`, `float`, `string`, `list` and `map` defaults in XML configurations.
8+
* Added support for UTF-8 requirements
89

910
2.8.0
1011
-----

‎src/Symfony/Component/Routing/Generator/UrlGenerator.php

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ protected function doGenerate($variables, $defaults, $requirements, $tokens, $pa
158158
if ('variable' === $token[0]) {
159159
if (!$optional || !array_key_exists($token[3], $defaults) || null !== $mergedParams[$token[3]] && (string) $mergedParams[$token[3]] !== (string) $defaults[$token[3]]) {
160160
// check requirement
161-
if (null !== $this->strictRequirements && !preg_match('#^'.$token[2].'$#', $mergedParams[$token[3]])) {
161+
if (null !== $this->strictRequirements && !preg_match('#^'.$token[2].'$#'.(empty($token[4]) ? '' : 'u'), $mergedParams[$token[3]])) {
162162
if ($this->strictRequirements) {
163163
throw new InvalidParameterException(strtr($message, array('{parameter}' => $token[3], '{route}' => $name, '{expected}' => $token[2], '{given}' => $mergedParams[$token[3]])));
164164
}
@@ -212,7 +212,7 @@ protected function doGenerate($variables, $defaults, $requirements, $tokens, $pa
212212
$routeHost = '';
213213
foreach ($hostTokens as $token) {
214214
if ('variable' === $token[0]) {
215-
if (null !== $this->strictRequirements && !preg_match('#^'.$token[2].'$#i', $mergedParams[$token[3]])) {
215+
if (null !== $this->strictRequirements && !preg_match('#^'.$token[2].'$#i'.(empty($token[4]) ? '' : 'u'), $mergedParams[$token[3]])) {
216216
if ($this->strictRequirements) {
217217
throw new InvalidParameterException(strtr($message, array('{parameter}' => $token[3], '{route}' => $name, '{expected}' => $token[2], '{given}' => $mergedParams[$token[3]])));
218218
}

‎src/Symfony/Component/Routing/Matcher/Dumper/PhpMatcherDumper.php

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -223,8 +223,9 @@ private function compileRoute(Route $route, $name, $supportsRedirections, $paren
223223
}
224224

225225
$supportsTrailingSlash = $supportsRedirections && (!$methods || in_array('HEAD', $methods));
226+
$regex = $compiledRoute->getRegex();
226227

227-
if (!count($compiledRoute->getPathVariables()) && false !== preg_match('#^(.)\^(?P<url>.*?)\$\1#', $compiledRoute->getRegex(), $m)) {
228+
if (!count($compiledRoute->getPathVariables()) && false !== preg_match('#^(.)\^(?P<url>.*?)\$\1#'.(substr($regex, -1) === 'u' ? 'u' : ''), $regex, $m)) {
228229
if ($supportsTrailingSlash && substr($m['url'], -1) === '/') {
229230
$conditions[] = sprintf("rtrim(\$pathinfo, '/') === %s", var_export(rtrim(str_replace('\\', '', $m['url']), '/'), true));
230231
$hasTrailingSlash = true;
@@ -236,7 +237,6 @@ private function compileRoute(Route $route, $name, $supportsRedirections, $paren
236237
$conditions[] = sprintf('0 === strpos($pathinfo, %s)', var_export($compiledRoute->getStaticPrefix(), true));
237238
}
238239

239-
$regex = $compiledRoute->getRegex();
240240
if ($supportsTrailingSlash && $pos = strpos($regex, '/$')) {
241241
$regex = substr($regex, 0, $pos).'/?$'.substr($regex, $pos + 2);
242242
$hasTrailingSlash = true;

‎src/Symfony/Component/Routing/Route.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ class Route implements \Serializable
7070
* Available options:
7171
*
7272
* * compiler_class: A class name able to compile this route instance (RouteCompiler by default)
73+
* * utf8: Whether UTF-8 matching is enforced ot not
7374
*
7475
* @param string $path The path pattern to match
7576
* @param array $defaults An array of default parameter values

‎src/Symfony/Component/Routing/RouteCompiler.php

Lines changed: 53 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,16 @@ private static function compilePattern(Route $route, $pattern, $isHost)
9191
$matches = array();
9292
$pos = 0;
9393
$defaultSeparator = $isHost ? '.' : '/';
94+
$useUtf8 = preg_match('//u', $pattern);
95+
$needsUtf8 = $route->getOption('utf8');
96+
97+
if (!$needsUtf8 && $useUtf8 && preg_match('/[\x80-\xFF]/', $pattern)) {
98+
$needsUtf8 = true;
99+
@trigger_error(sprintf('Using UTF-8 route patterns without setting the "utf8" option is deprecated since Symfony 3.2 and will throw a LogicException in 4.0. Turn on the "utf8" route option for pattern "%s".', $pattern), E_USER_DEPRECATED);
100+
}
101+
if (!$useUtf8 && $needsUtf8) {
102+
throw new \LogicException(sprintf('Cannot mix UTF-8 requirements with non-UTF-8 pattern "%s".', $pattern));
103+
}
94104

95105
// Match all variables enclosed in "{}" and iterate over them. But we only want to match the innermost variable
96106
// in case of nested "{}", e.g. {foo{bar}}. This in ensured because \w does not match "{" or "}" itself.
@@ -100,7 +110,15 @@ private static function compilePattern(Route $route, $pattern, $isHost)
100110
// get all static text preceding the current variable
101111
$precedingText = substr($pattern, $pos, $match[0][1] - $pos);
102112
$pos = $match[0][1] + strlen($match[0][0]);
103-
$precedingChar = strlen($precedingText) > 0 ? substr($precedingText, -1) : '';
113+
114+
if (!strlen($precedingText)) {
115+
$precedingChar = '';
116+
} elseif ($useUtf8) {
117+
preg_match('/.$/u', $precedingText, $precedingChar);
118+
$precedingChar = $precedingChar[0];
119+
} else {
120+
$precedingChar = substr($precedingText, -1);
121+
}
104122
$isSeparator = '' !== $precedingChar && false !== strpos(static::SEPARATORS, $precedingChar);
105123

106124
if (is_numeric($varName)) {
@@ -110,8 +128,8 @@ private static function compilePattern(Route $route, $pattern, $isHost)
110128
throw new \LogicException(sprintf('Route pattern "%s" cannot reference variable name "%s" more than once.', $pattern, $varName));
111129
}
112130

113-
if ($isSeparator && strlen($precedingText) > 1) {
114-
$tokens[] = array('text', substr($precedingText, 0, -1));
131+
if ($isSeparator && $precedingText !== $precedingChar) {
132+
$tokens[] = array('text', substr($precedingText, 0, -strlen($precedingChar)));
115133
} elseif (!$isSeparator && strlen($precedingText) > 0) {
116134
$tokens[] = array('text', $precedingText);
117135
}
@@ -126,7 +144,7 @@ private static function compilePattern(Route $route, $pattern, $isHost)
126144
// If {page} would also match the separating dot, {_format} would never match as {page} will eagerly consume everything.
127145
// Also even if {_format} was not optional the requirement prevents that {page} matches something that was originally
128146
// part of {_format} when generating the URL, e.g. _format = 'mobile.html'.
129-
$nextSeparator = self::findNextSeparator($followingPattern);
147+
$nextSeparator = self::findNextSeparator($followingPattern, $useUtf8);
130148
$regexp = sprintf(
131149
'[^%s%s]+',
132150
preg_quote($defaultSeparator, self::REGEX_DELIMITER),
@@ -140,6 +158,16 @@ private static function compilePattern(Route $route, $pattern, $isHost)
140158
// directly adjacent, e.g. '/{x}{y}'.
141159
$regexp .= '+';
142160
}
161+
} else {
162+
if (!preg_match('//u', $regexp)) {
163+
$useUtf8 = false;
164+
} elseif (!$needsUtf8 && preg_match('/[\x80-\xFF]|(?<!\\\\)\\\\(?:\\\\\\\\)*+(?-i:X|[pP][\{CLMNPSZ]|x\{[A-Fa-f0-9]{3})/', $regexp)) {
165+
$needsUtf8 = true;
166+
@trigger_error(sprintf('Using UTF-8 route requirements without setting the "utf8" option is deprecated since Symfony 3.2 and will throw a LogicException in 4.0. Turn on the "utf8" route option for variable "%s" in pattern "%s".', $varName, $pattern), E_USER_DEPRECATED);
167+
}
168+
if (!$useUtf8 && $needsUtf8) {
169+
throw new \LogicException(sprintf('Cannot mix UTF-8 requirement with non-UTF-8 charset for variable "%s" in pattern "%s".', $varName, $pattern));
170+
}
143171
}
144172

145173
$tokens[] = array('variable', $isSeparator ? $precedingChar : '', $regexp, $varName);
@@ -168,10 +196,21 @@ private static function compilePattern(Route $route, $pattern, $isHost)
168196
for ($i = 0, $nbToken = count($tokens); $i < $nbToken; ++$i) {
169197
$regexp .= self::computeRegexp($tokens, $i, $firstOptional);
170198
}
199+
$regexp = self::REGEX_DELIMITER.'^'.$regexp.'$'.self::REGEX_DELIMITER.'s'.($isHost ? 'i' : '');
200+
201+
// enable Utf8 matching if really required
202+
if ($needsUtf8) {
203+
$regexp .= 'u';
204+
for ($i = 0, $nbToken = count($tokens); $i < $nbToken; ++$i) {
205+
if ('variable' === $tokens[$i][0]) {
206+
$tokens[$i][] = true;
207+
}
208+
}
209+
}
171210

172211
return array(
173212
'staticPrefix' => 'text' === $tokens[0][0] ? $tokens[0][1] : '',
174-
'regex' => self::REGEX_DELIMITER.'^'.$regexp.'$'.self::REGEX_DELIMITER.'s'.($isHost ? 'i' : ''),
213+
'regex' => $regexp,
175214
'tokens' => array_reverse($tokens),
176215
'variables' => $variables,
177216
);
@@ -181,19 +220,25 @@ private static function compilePattern(Route $route, $pattern, $isHost)
181220
* Returns the next static character in the Route pattern that will serve as a separator.
182221
*
183222
* @param string $pattern The route pattern
223+
* @param bool $useUtf8 Whether the character is encoded in UTF-8 or not
184224
*
185225
* @return string The next static character that functions as separator (or empty string when none available)
186226
*/
187-
private static function findNextSeparator($pattern)
227+
private static function findNextSeparator($pattern, $useUtf8)
188228
{
189229
if ('' == $pattern) {
190230
// return empty string if pattern is empty or false (false which can be returned by substr)
191231
return '';
192232
}
193233
// first remove all placeholders from the pattern so we can find the next real static character
194-
$pattern = preg_replace('#\{\w+\}#', '', $pattern);
234+
if ('' === $pattern = preg_replace('#\{\w+\}#', '', $pattern)) {
235+
return '';
236+
}
237+
if ($useUtf8) {
238+
preg_match('/^./u', $pattern, $pattern);
239+
}
195240

196-
return isset($pattern[0]) && false !== strpos(static::SEPARATORS, $pattern[0]) ? $pattern[0] : '';
241+
return false !== strpos(static::SEPARATORS, $pattern[0]) ? $pattern[0] : '';
197242
}
198243

199244
/**

‎src/Symfony/Component/Routing/Tests/Generator/UrlGeneratorTest.php

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,15 @@ public function testGenerateForRouteWithInvalidMandatoryParameter()
233233
$this->getGenerator($routes)->generate('test', array('foo' => 'bar'), UrlGeneratorInterface::ABSOLUTE_URL);
234234
}
235235

236+
/**
237+
* @expectedException \Symfony\Component\Routing\Exception\InvalidParameterException
238+
*/
239+
public function testGenerateForRouteWithInvalidUtf8Parameter()
240+
{
241+
$routes = $this->getRoutes('test', new Route('/testing/{foo}', array(), array('foo' => '\pL+'), array('utf8' => true)));
242+
$this->getGenerator($routes)->generate('test', array('foo' => 'abc123'), UrlGeneratorInterface::ABSOLUTE_URL);
243+
}
244+
236245
/**
237246
* @expectedException \Symfony\Component\Routing\Exception\InvalidParameterException
238247
*/

‎src/Symfony/Component/Routing/Tests/Matcher/UrlMatcherTest.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ public function testMatchNonAlpha()
165165
{
166166
$collection = new RouteCollection();
167167
$chars = '!"$%éà &\'()*+,./:;<=>@ABCDEFGHIJKLMNOPQRSTUVWXYZ\\[]^_`abcdefghijklmnopqrstuvwxyz{|}~-';
168-
$collection->add('foo', new Route('/{foo}/bar', array(), array('foo' => '['.preg_quote($chars).']+')));
168+
$collection->add('foo', new Route('/{foo}/bar', array(), array('foo' => '['.preg_quote($chars).']+'), array('utf8' => true)));
169169

170170
$matcher = new UrlMatcher($collection, new RequestContext());
171171
$this->assertEquals(array('_route' => 'foo', 'foo' => $chars), $matcher->match('/'.rawurlencode($chars).'/bar'));

‎src/Symfony/Component/Routing/Tests/RouteCompilerTest.php

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@
1111

1212
namespace Symfony\Component\Routing\Tests;
1313

14+
use Symfony\Bridge\PhpUnit\ErrorAssert;
1415
use Symfony\Component\Routing\Route;
16+
use Symfony\Component\Routing\RouteCompiler;
1517

1618
class RouteCompilerTest extends \PHPUnit_Framework_TestCase
1719
{
@@ -162,6 +164,87 @@ public function provideCompileData()
162164
array('text', '/foo'),
163165
),
164166
),
167+
168+
array(
169+
'Static non UTF-8 route',
170+
array("/fo\xE9"),
171+
"/fo\xE9", "#^/fo\xE9$#s", array(), array(
172+
array('text', "/fo\xE9"),
173+
),
174+
),
175+
176+
array(
177+
'Route with an explicit UTF-8 requirement',
178+
array('/{bar}', array('bar' => null), array('bar' => '.'), array('utf8' => true)),
179+
'', '#^/(?P<bar>.)?$#su', array('bar'), array(
180+
array('variable', '/', '.', 'bar', true),
181+
),
182+
),
183+
);
184+
}
185+
186+
/**
187+
* @dataProvider provideCompileImplicitUtf8Data
188+
* @requires function Symfony\Bridge\PhpUnit\ErrorAssert::assertDeprecationsAreTriggered
189+
*/
190+
public function testCompileImplicitUtf8Data($name, $arguments, $prefix, $regex, $variables, $tokens, $deprecationType)
191+
{
192+
$deprecations = array(
193+
sprintf('Using UTF-8 route %s without setting the "utf8" option is deprecated', $deprecationType),
194+
);
195+
196+
ErrorAssert::assertDeprecationsAreTriggered($deprecations, function () use ($name, $arguments, $prefix, $regex, $variables, $tokens) {
197+
$r = new \ReflectionClass('Symfony\\Component\\Routing\\Route');
198+
$route = $r->newInstanceArgs($arguments);
199+
200+
$compiled = $route->compile();
201+
$this->assertEquals($prefix, $compiled->getStaticPrefix(), $name.' (static prefix)');
202+
$this->assertEquals($regex, $compiled->getRegex(), $name.' (regex)');
203+
$this->assertEquals($variables, $compiled->getVariables(), $name.' (variables)');
204+
$this->assertEquals($tokens, $compiled->getTokens(), $name.' (tokens)');
205+
});
206+
}
207+
208+
public function provideCompileImplicitUtf8Data()
209+
{
210+
return array(
211+
array(
212+
'Static UTF-8 route',
213+
array('/foé'),
214+
'/foé', '#^/foé$#su', array(), array(
215+
array('text', '/foé'),
216+
),
217+
'patterns',
218+
),
219+
220+
array(
221+
'Route with an implicit UTF-8 requirement',
222+
array('/{bar}', array('bar' => null), array('bar' => 'é')),
223+
'', '#^/(?P<bar>é)?$#su', array('bar'), array(
224+
array('variable', '/', 'é', 'bar', true),
225+
),
226+
'requirements',
227+
),
228+
229+
array(
230+
'Route with a UTF-8 class requirement',
231+
array('/{bar}', array('bar' => null), array('bar' => '\pM')),
232+
'', '#^/(?P<bar>\pM)?$#su', array('bar'), array(
233+
array('variable', '/', '\pM', 'bar', true),
234+
),
235+
'requirements',
236+
),
237+
238+
array(
239+
'Route with a UTF-8 separator',
240+
array('/foo/{bar}§{_format}', array(), array(), array('compiler_class' => Utf8RouteCompiler::class)),
241+
'/foo', '#^/foo/(?P<bar>[^/§]++)§(?P<_format>[^/]++)$#su', array('bar', '_format'), array(
242+
array('variable', '§', '[^/]++', '_format', true),
243+
array('variable', '/', '[^/§]++', 'bar', true),
244+
array('text', '/foo'),
245+
),
246+
'patterns',
247+
),
165248
);
166249
}
167250

@@ -175,6 +258,26 @@ public function testRouteWithSameVariableTwice()
175258
$compiled = $route->compile();
176259
}
177260

261+
/**
262+
* @expectedException \LogicException
263+
*/
264+
public function testRouteCharsetMismatch()
265+
{
266+
$route = new Route("/\xE9/{bar}", array(), array('bar' => '.'), array('utf8' => true));
267+
268+
$compiled = $route->compile();
269+
}
270+
271+
/**
272+
* @expectedException \LogicException
273+
*/
274+
public function testRequirementCharsetMismatch()
275+
{
276+
$route = new Route('/foo/{bar}', array(), array('bar' => "\xE9"), array('utf8' => true));
277+
278+
$compiled = $route->compile();
279+
}
280+
178281
/**
179282
* @expectedException \InvalidArgumentException
180283
*/
@@ -275,3 +378,8 @@ public function provideCompileWithHostData()
275378
);
276379
}
277380
}
381+
382+
class Utf8RouteCompiler extends RouteCompiler
383+
{
384+
const SEPARATORS = '/§';
385+
}

0 commit comments

Comments
 (0)