Support decimal extended type

.gitignore

@@ -17,3 +17,5 @@ sophia
 # Vim Swap files
 *.sw[a-z]
 .idea
+
+venv/*

CHANGELOG.md

@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## Unreleased
 
 ### Added
+- Decimal type support (#203).
 
 ### Changed
 - Bump msgpack requirement to 1.0.4 (PR #223).

tarantool/error.py

@@ -109,10 +109,20 @@ class ConfigurationError(Error):
     Error of initialization with a user-provided configuration.
     '''
 
+class MsgpackError(Error):
+    '''
+    Error with encoding or decoding of MP_EXT types
+    '''
+
+class MsgpackWarning(UserWarning):
+    '''
+    Warning with encoding or decoding of MP_EXT types
+    '''
 
 __all__ = ("Warning", "Error", "InterfaceError", "DatabaseError", "DataError",
            "OperationalError", "IntegrityError", "InternalError",
-           "ProgrammingError", "NotSupportedError")
+           "ProgrammingError", "NotSupportedError", "MsgpackError",
+           "MsgpackWarning")
 
 # Monkey patch os.strerror for win32
 if sys.platform == "win32":

tarantool/msgpack_ext/decimal.py

@@ -0,0 +1,228 @@
+from decimal import Decimal
+
+from tarantool.error import MsgpackError, MsgpackWarning, warn
+
+# https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-decimal-type
+#
+# The decimal MessagePack representation looks like this:
+# +--------+-------------------+------------+===============+
+# | MP_EXT | length (optional) | MP_DECIMAL | PackedDecimal |
+# +--------+-------------------+------------+===============+
+#
+# PackedDecimal has the following structure:
+#
+#  <--- length bytes -->
+# +-------+=============+
+# | scale |     BCD     |
+# +-------+=============+
+#
+# Here scale is either MP_INT or MP_UINT.
+# scale = number of digits after the decimal point
+#
+# BCD is a sequence of bytes representing decimal digits of the encoded number
+# (each byte has two decimal digits each encoded using 4-bit nibbles), so
+# byte >> 4 is the first digit  and byte & 0x0f is the second digit. The
+# leftmost digit in the array is the most significant. The rightmost digit in
+# the array is the least significant.
+#
+# The first byte of the BCD array contains the first digit of the number,
+# represented as follows:
+#
+# |  4 bits           |  4 bits           |
+#    = 0x                = the 1st digit
+#
+# (The first nibble contains 0 if the decimal number has an even number of
+# digits.) The last byte of the BCD array contains the last digit of the number
+# and the final nibble, represented as follows:
+#
+# |  4 bits           |  4 bits           |
+#    = the last digit    = nibble
+#
+# The final nibble represents the number’s sign:
+#
+#     0x0a, 0x0c, 0x0e, 0x0f stand for plus,
+#     0x0b and 0x0d stand for minus.
+
+EXT_ID = 1
+
+TARANTOOL_DECIMAL_MAX_DIGITS = 38
+
+def get_mp_sign(sign):
+    if sign == '+':
+        return 0x0c
+
+    if sign == '-':
+        return 0x0d
+
+    raise RuntimeError
+
+def add_mp_digit(digit, bytes_reverted, digit_count):
+    if digit_count % 2 == 0:
+        bytes_reverted[-1] = bytes_reverted[-1] | (digit << 4)
+    else:
+        bytes_reverted.append(digit)
+
+def check_valid_tarantool_decimal(str_repr, scale, first_digit_ind):
+# Decimal numbers have 38 digits of precision, that is, the total number of
+# digits before and after the decimal point can be 38. If there are more
+# digits arter the decimal point, the precision is lost. If there are more
+# digits before the decimal point, error is thrown.
+#
+# Tarantool 2.10.1-0-g482d91c66
+#
+# tarantool> decimal.new('10000000000000000000000000000000000000')
+# ---
+# - 10000000000000000000000000000000000000
+# ...
+#
+# tarantool> decimal.new('100000000000000000000000000000000000000')
+# ---
+# - error: '[string "return VERSION"]:1: variable ''VERSION'' is not declared'
+# ...
+#
+# tarantool> decimal.new('1.0000000000000000000000000000000000001')
+# ---
+# - 1.0000000000000000000000000000000000001
+# ...
+#
+# tarantool> decimal.new('1.00000000000000000000000000000000000001')
+# ---
+# - 1.0000000000000000000000000000000000000
+# ...
+#
+# In fact, there is also an exceptional case: if decimal starts with `0.`,
+# 38 digits after the decimal point are supported without the loss of precision. 
+#
+# tarantool> decimal.new('0.00000000000000000000000000000000000001')
+# ---
+# - 0.00000000000000000000000000000000000001
+# ...
+#
+# tarantool> decimal.new('0.000000000000000000000000000000000000001')
+# ---
+# - 0.00000000000000000000000000000000000000
+# ...
+    if scale > 0:
+        digit_count = len(str_repr) - 1 - first_digit_ind
+    else:
+        digit_count = len(str_repr) - first_digit_ind
+
+    if digit_count <= TARANTOOL_DECIMAL_MAX_DIGITS:
+        return True
+
+    if (digit_count - scale) > TARANTOOL_DECIMAL_MAX_DIGITS:
+        raise MsgpackError('Decimal cannot be encoded: Tarantool decimal ' + \
+                           'supports a maximum of 38 digits.')
+
+    starts_with_zero = str_repr[first_digit_ind] == '0'
+
+    if (    (digit_count > TARANTOOL_DECIMAL_MAX_DIGITS + 1) or \
+            (digit_count == TARANTOOL_DECIMAL_MAX_DIGITS + 1 \
+            and not starts_with_zero)):
+        warn('Decimal encoded with loss of precision: ' + \
+             'Tarantool decimal supports a maximum of 38 digits.',
+             MsgpackWarning)
+        return False
+
+    return True
+
+def strip_decimal_str(str_repr, scale, first_digit_ind):
+    assert scale > 0
+    # Strip extra bytes
+    str_repr = str_repr[:TARANTOOL_DECIMAL_MAX_DIGITS + 1 + first_digit_ind]
+
+    str_repr = str_repr.rstrip('0')
+    str_repr = str_repr.rstrip('.')
+    # Do not strips zeroes before the decimal point
+    return str_repr
+
+def encode(obj):
+    # Non-scientific string with trailing zeroes removed
+    str_repr = format(obj, 'f')
+
+    bytes_reverted = bytearray()
+
+    scale = 0
+    for i in range(len(str_repr)):
+        str_digit = str_repr[i]
+        if str_digit == '.':
+            scale = len(str_repr) - i - 1
+            break
+
+    if str_repr[0] == '-':
+        sign = '-'
+        first_digit_ind = 1
+    else:
+        sign = '+'
+        first_digit_ind = 0
+
+    if not check_valid_tarantool_decimal(str_repr, scale, first_digit_ind):
+        str_repr = strip_decimal_str(str_repr, scale, first_digit_ind)
+
+    bytes_reverted.append(get_mp_sign(sign))
+
+    digit_count = 0
+    # We need to update the scale after possible strip_decimal_str() 
+    scale = 0
+
+    for i in range(len(str_repr) - 1, first_digit_ind - 1, -1):
+        str_digit = str_repr[i]
+        if str_digit == '.':
+            scale = len(str_repr) - i - 1
+            continue
+
+        add_mp_digit(int(str_digit), bytes_reverted, digit_count)
+        digit_count = digit_count + 1
+
+    # Remove leading zeroes since they already covered by scale
+    for i in range(len(bytes_reverted) - 1, 0, -1):
+        if bytes_reverted[i] != 0:
+            break
+        bytes_reverted.pop()
+
+    bytes_reverted.append(scale)
+
+    return bytes(bytes_reverted[::-1])
+
+
+def get_str_sign(nibble):
+    if nibble == 0x0a or nibble == 0x0c or nibble == 0x0e or nibble == 0x0f:
+        return '+'
+
+    if nibble == 0x0b or nibble == 0x0d:
+        return '-'
+
+    raise MsgpackError('Unexpected MP_DECIMAL sign nibble')
+
+def add_str_digit(digit, digits_reverted, scale):
+    if not (0 <= digit <= 9):
+        raise MsgpackError('Unexpected MP_DECIMAL digit nibble')
+
+    if len(digits_reverted) == scale:
+        digits_reverted.append('.')
+
+    digits_reverted.append(str(digit))
+
+def decode(data):
+    scale = data[0] 
+
+    sign = get_str_sign(data[-1] & 0x0f)
+
+    # Parse from tail since scale is counted from the tail.
+    digits_reverted = []
+
+    add_str_digit(data[-1] >> 4, digits_reverted, scale)
+
+    for i in range(len(data) - 2, 0, -1):
+        add_str_digit(data[i] & 0x0f, digits_reverted, scale)
+        add_str_digit(data[i] >> 4, digits_reverted, scale)
+
+    # Add leading zeroes in case of 0.000... number
+    for i in range(len(digits_reverted), scale + 1):
+        add_str_digit(0, digits_reverted, scale)
+
+    digits_reverted.append(sign)
+
+    str_repr = ''.join(digits_reverted[::-1])
+
+    return Decimal(str_repr)

tarantool/msgpack_ext/packer.py

@@ -0,0 +1,9 @@
+from decimal import Decimal
+from msgpack import ExtType
+
+import tarantool.msgpack_ext.decimal as ext_decimal
+
+def default(obj):
+    if isinstance(obj, Decimal):
+        return ExtType(ext_decimal.EXT_ID, ext_decimal.encode(obj))
+    raise TypeError("Unknown type: %r" % (obj,))

tarantool/msgpack_ext/unpacker.py

@@ -0,0 +1,6 @@
+import tarantool.msgpack_ext.decimal as ext_decimal
+
+def ext_hook(code, data):
+    if code == ext_decimal.EXT_ID:
+        return ext_decimal.decode(data)
+    raise NotImplementedError("Unknown msgpack type: %d" % (code,))

tarantool/request.py

@@ -59,6 +59,8 @@
     binary_types
 )
 
+from tarantool.msgpack_ext.packer import default as packer_default
+
 class Request(object):
     '''
     Represents a single request to the server in compliance with the
@@ -122,6 +124,8 @@ def __init__(self, conn):
         else:
             packer_kwargs['use_bin_type'] = True
 
+        packer_kwargs['default'] = packer_default
+
         self.packer = msgpack.Packer(**packer_kwargs)
 
     def _dumps(self, src):

tarantool/response.py

@@ -29,6 +29,7 @@
     tnt_strerror
 )
 
+from tarantool.msgpack_ext.unpacker import ext_hook as unpacker_ext_hook
 
 class Response(Sequence):
     '''
@@ -86,6 +87,8 @@ def __init__(self, conn, response):
         if msgpack.version >= (1, 0, 0):
             unpacker_kwargs['strict_map_key'] = False
 
+        unpacker_kwargs['ext_hook'] = unpacker_ext_hook
+
         unpacker = msgpack.Unpacker(**unpacker_kwargs)
 
         unpacker.feed(response)

test/suites/__init__.py

@@ -15,12 +15,14 @@
 from .test_dbapi import TestSuite_DBAPI
 from .test_encoding import TestSuite_Encoding
 from .test_ssl import TestSuite_Ssl
+from .test_msgpack_ext import TestSuite_MsgpackExt
 
 test_cases = (TestSuite_Schema_UnicodeConnection,
               TestSuite_Schema_BinaryConnection,
               TestSuite_Request, TestSuite_Protocol, TestSuite_Reconnect,
               TestSuite_Mesh, TestSuite_Execute, TestSuite_DBAPI,
-              TestSuite_Encoding, TestSuite_Pool, TestSuite_Ssl)
+              TestSuite_Encoding, TestSuite_Pool, TestSuite_Ssl,
+              TestSuite_MsgpackExt)
 
 def load_tests(loader, tests, pattern):
     suite = unittest.TestSuite()

test/suites/lib/skip.py

@@ -43,6 +43,38 @@ def wrapper(self, *args, **kwargs):
 
     return wrapper
 
+def skip_or_run_test_pcall_require(func, REQUIRED_TNT_MODULE, msg):
+    """Decorator to skip or run tests depending on tarantool
+    module requre success or fail.
+
+    Also, it can be used with the 'setUp' method for skipping
+    the whole test suite.
+    """
+
+    @functools.wraps(func)
+    def wrapper(self, *args, **kwargs):
+        if func.__name__ == 'setUp':
+            func(self, *args, **kwargs)
+
+        srv = None
+
+        if hasattr(self, 'servers'):
+            srv = self.servers[0]
+
+        if hasattr(self, 'srv'):
+            srv = self.srv
+
+        assert srv is not None
+
+        resp = srv.admin("pcall(require, '%s')" % REQUIRED_TNT_MODULE)
+        if not resp[0]:
+            self.skipTest('Tarantool %s' % (msg, ))
+
+        if func.__name__ != 'setUp':
+            func(self, *args, **kwargs)
+
+    return wrapper
+
 
 def skip_or_run_test_python(func, REQUIRED_PYTHON_VERSION, msg):
     """Decorator to skip or run tests depending on the Python version.
@@ -101,3 +133,13 @@ def skip_or_run_conn_pool_test(func):
     return skip_or_run_test_python(func, '3.7',
                                    'does not support ConnectionPool')
 
+def skip_or_run_decimal_test(func):
+    """Decorator to skip or run decimal-related tests depending on
+    the tarantool version.
+
+    Tarantool supports decimal type only since 2.2.1 version.
+    See https://github.com/tarantool/tarantool/issues/692
+    """
+
+    return skip_or_run_test_pcall_require(func, 'decimal',
+                                      'does not support decimal type')