Improve performance of lazy validation message formatting (encode#6709)

Backport to 3.9.x branch

Author: bluetech

requirements/requirements-testing.txt

@@ -2,3 +2,4 @@
 pytest==4.3.0
 pytest-django==3.4.8
 pytest-cov==2.6.1
+mock==3.0.5; python_version < '3.0'

rest_framework/compat.py

@@ -8,7 +8,6 @@
 import sys
 
 from django.conf import settings
-from django.core import validators
 from django.utils import six
 from django.views.generic import View
 
@@ -299,34 +298,5 @@ def md_filter_add_syntax_highlight(md):
     INDENT_SEPARATORS = (b',', b': ')
 
 
-class CustomValidatorMessage(object):
-    """
-    We need to avoid evaluation of `lazy` translated `message` in `django.core.validators.BaseValidator.__init__`.
-    https://github.com/django/django/blob/75ed5900321d170debef4ac452b8b3cf8a1c2384/django/core/validators.py#L297
-
-    Ref: https://github.com/encode/django-rest-framework/pull/5452
-    """
-
-    def __init__(self, *args, **kwargs):
-        self.message = kwargs.pop('message', self.message)
-        super(CustomValidatorMessage, self).__init__(*args, **kwargs)
-
-
-class MinValueValidator(CustomValidatorMessage, validators.MinValueValidator):
-    pass
-
-
-class MaxValueValidator(CustomValidatorMessage, validators.MaxValueValidator):
-    pass
-
-
-class MinLengthValidator(CustomValidatorMessage, validators.MinLengthValidator):
-    pass
-
-
-class MaxLengthValidator(CustomValidatorMessage, validators.MaxLengthValidator):
-    pass
-
-
 # Version Constants.
 PY36 = sys.version_info >= (3, 6)

rest_framework/fields.py

@@ -13,7 +13,8 @@
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.exceptions import ValidationError as DjangoValidationError
 from django.core.validators import (
-    EmailValidator, RegexValidator, URLValidator, ip_address_validators
+    EmailValidator, MaxLengthValidator, MaxValueValidator, MinLengthValidator,
+    MinValueValidator, RegexValidator, URLValidator, ip_address_validators
 )
 from django.forms import FilePathField as DjangoFilePathField
 from django.forms import ImageField as DjangoImageField
@@ -24,21 +25,19 @@
 from django.utils.duration import duration_string
 from django.utils.encoding import is_protected_type, smart_text
 from django.utils.formats import localize_input, sanitize_separators
-from django.utils.functional import lazy
 from django.utils.ipv6 import clean_ipv6_address
 from django.utils.timezone import utc
 from django.utils.translation import ugettext_lazy as _
 from pytz.exceptions import InvalidTimeError
 
 from rest_framework import ISO_8601
 from rest_framework.compat import (
-    Mapping, MaxLengthValidator, MaxValueValidator, MinLengthValidator,
-    MinValueValidator, ProhibitNullCharactersValidator, unicode_repr,
-    unicode_to_repr
+    Mapping, ProhibitNullCharactersValidator, unicode_repr, unicode_to_repr
 )
 from rest_framework.exceptions import ErrorDetail, ValidationError
 from rest_framework.settings import api_settings
 from rest_framework.utils import html, humanize_datetime, json, representation
+from rest_framework.utils.formatting import lazy_format
 
 
 class empty:
@@ -766,15 +765,11 @@ def __init__(self, **kwargs):
         self.min_length = kwargs.pop('min_length', None)
         super(CharField, self).__init__(**kwargs)
         if self.max_length is not None:
-            message = lazy(
-                self.error_messages['max_length'].format,
-                six.text_type)(max_length=self.max_length)
+            message = lazy_format(self.error_messages['max_length'], max_length=self.max_length)
             self.validators.append(
                 MaxLengthValidator(self.max_length, message=message))
         if self.min_length is not None:
-            message = lazy(
-                self.error_messages['min_length'].format,
-                six.text_type)(min_length=self.min_length)
+            message = lazy_format(self.error_messages['min_length'], min_length=self.min_length)
             self.validators.append(
                 MinLengthValidator(self.min_length, message=message))
 
@@ -935,15 +930,11 @@ def __init__(self, **kwargs):
         self.min_value = kwargs.pop('min_value', None)
         super(IntegerField, self).__init__(**kwargs)
         if self.max_value is not None:
-            message = lazy(
-                self.error_messages['max_value'].format,
-                six.text_type)(max_value=self.max_value)
+            message = lazy_format(self.error_messages['max_value'], max_value=self.max_value)
             self.validators.append(
                 MaxValueValidator(self.max_value, message=message))
         if self.min_value is not None:
-            message = lazy(
-                self.error_messages['min_value'].format,
-                six.text_type)(min_value=self.min_value)
+            message = lazy_format(self.error_messages['min_value'], min_value=self.min_value)
             self.validators.append(
                 MinValueValidator(self.min_value, message=message))
 
@@ -975,15 +966,11 @@ def __init__(self, **kwargs):
         self.min_value = kwargs.pop('min_value', None)
         super(FloatField, self).__init__(**kwargs)
         if self.max_value is not None:
-            message = lazy(
-                self.error_messages['max_value'].format,
-                six.text_type)(max_value=self.max_value)
+            message = lazy_format(self.error_messages['max_value'], max_value=self.max_value)
             self.validators.append(
                 MaxValueValidator(self.max_value, message=message))
         if self.min_value is not None:
-            message = lazy(
-                self.error_messages['min_value'].format,
-                six.text_type)(min_value=self.min_value)
+            message = lazy_format(self.error_messages['min_value'], min_value=self.min_value)
             self.validators.append(
                 MinValueValidator(self.min_value, message=message))
 
@@ -1034,15 +1021,11 @@ def __init__(self, max_digits, decimal_places, coerce_to_string=None, max_value=
         super(DecimalField, self).__init__(**kwargs)
 
         if self.max_value is not None:
-            message = lazy(
-                self.error_messages['max_value'].format,
-                six.text_type)(max_value=self.max_value)
+            message = lazy_format(self.error_messages['max_value'], max_value=self.max_value)
             self.validators.append(
                 MaxValueValidator(self.max_value, message=message))
         if self.min_value is not None:
-            message = lazy(
-                self.error_messages['min_value'].format,
-                six.text_type)(min_value=self.min_value)
+            message = lazy_format(self.error_messages['min_value'], min_value=self.min_value)
             self.validators.append(
                 MinValueValidator(self.min_value, message=message))
 
@@ -1380,15 +1363,11 @@ def __init__(self, **kwargs):
         self.min_value = kwargs.pop('min_value', None)
         super(DurationField, self).__init__(**kwargs)
         if self.max_value is not None:
-            message = lazy(
-                self.error_messages['max_value'].format,
-                six.text_type)(max_value=self.max_value)
+            message = lazy_format(self.error_messages['max_value'], max_value=self.max_value)
             self.validators.append(
                 MaxValueValidator(self.max_value, message=message))
         if self.min_value is not None:
-            message = lazy(
-                self.error_messages['min_value'].format,
-                six.text_type)(min_value=self.min_value)
+            message = lazy_format(self.error_messages['min_value'], min_value=self.min_value)
             self.validators.append(
                 MinValueValidator(self.min_value, message=message))
 
@@ -1633,10 +1612,10 @@ def __init__(self, *args, **kwargs):
         super(ListField, self).__init__(*args, **kwargs)
         self.child.bind(field_name='', parent=self)
         if self.max_length is not None:
-            message = self.error_messages['max_length'].format(max_length=self.max_length)
+            message = lazy_format(self.error_messages['max_length'], max_length=self.max_length)
             self.validators.append(MaxLengthValidator(self.max_length, message=message))
         if self.min_length is not None:
-            message = self.error_messages['min_length'].format(min_length=self.min_length)
+            message = lazy_format(self.error_messages['min_length'], min_length=self.min_length)
             self.validators.append(MinLengthValidator(self.min_length, message=message))
 
     def get_value(self, dictionary):
@@ -1907,9 +1886,7 @@ def __init__(self, model_field, **kwargs):
         max_length = kwargs.pop('max_length', None)
         super(ModelField, self).__init__(**kwargs)
         if max_length is not None:
-            message = lazy(
-                self.error_messages['max_length'].format,
-                six.text_type)(max_length=self.max_length)
+            message = lazy_format(self.error_messages['max_length'], max_length=self.max_length)
             self.validators.append(
                 MaxLengthValidator(self.max_length, message=message))
 

rest_framework/utils/formatting.py

@@ -5,7 +5,8 @@
 
 import re
 
-from django.utils.encoding import force_text
+from django.utils import six
+from django.utils.encoding import force_text, python_2_unicode_compatible
 from django.utils.html import escape
 from django.utils.safestring import mark_safe
 
@@ -67,3 +68,30 @@ def markup_description(description):
         description = escape(description).replace('\n', '<br />')
         description = '<p>' + description + '</p>'
     return mark_safe(description)
+
+
+@python_2_unicode_compatible
+class lazy_format(object):
+    """
+    Delay formatting until it's actually needed.
+
+    Useful when the format string or one of the arguments is lazy.
+
+    Not using Django's lazy because it is too slow.
+    """
+    __slots__ = ('format_string', 'args', 'kwargs', 'result')
+
+    def __init__(self, format_string, *args, **kwargs):
+        self.result = None
+        self.format_string = format_string
+        self.args = args
+        self.kwargs = kwargs
+
+    def __str__(self):
+        if self.result is None:
+            self.result = self.format_string.format(*self.args, **self.kwargs)
+            self.format_string, self.args, self.kwargs = None, None, None
+        return self.result
+
+    def __mod__(self, value):
+        return six.text_type(self) % value

tests/test_utils.py

@@ -1,6 +1,11 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+
 from django.conf.urls import url
 from django.test import TestCase, override_settings
 
@@ -9,6 +14,7 @@
 from rest_framework.serializers import ModelSerializer
 from rest_framework.utils import json
 from rest_framework.utils.breadcrumbs import get_breadcrumbs
+from rest_framework.utils.formatting import lazy_format
 from rest_framework.utils.urls import remove_query_param, replace_query_param
 from rest_framework.views import APIView
 from rest_framework.viewsets import ModelViewSet
@@ -260,3 +266,19 @@ def test_invalid_unicode(self):
         removed_key = 'page'
 
         assert key in remove_query_param(q, removed_key)
+
+
+class LazyFormatTests(TestCase):
+    def test_it_formats_correctly(self):
+        formatted = lazy_format('Does {} work? {answer}: %s', 'it', answer='Yes')
+        assert str(formatted) == 'Does it work? Yes: %s'
+        assert formatted % 'it does' == 'Does it work? Yes: it does'
+
+    def test_it_formats_lazily(self):
+        message = mock.Mock(wraps='message')
+        formatted = lazy_format(message)
+        assert message.format.call_count == 0
+        str(formatted)
+        assert message.format.call_count == 1
+        str(formatted)
+        assert message.format.call_count == 1