diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c8a4fd3e90..52bfcd00a5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,19 @@ +3.29.0 +====== +December 19, 2023 + +Features +-------- +* Add support for Python 3.9 through 3.12, drop support for 3.7 (PYTHON-1283) +* Removal of dependency on six module (PR 1172) +* Raise explicit exception when deserializing a vector with a subtype that isn’t a constant size (PYTHON-1371) + +Others +------ +* Remove outdated Python pre-3.7 references (PR 1186) +* Remove backup(.bak) files (PR 1185) +* Fix doc typo in add_callbacks (PR 1177) + 3.28.0 ====== June 5, 2023 diff --git a/benchmarks/base.py b/benchmarks/base.py index 47a03bbd68..2000b4069f 100644 --- a/benchmarks/base.py +++ b/benchmarks/base.py @@ -54,7 +54,7 @@ from cassandra.io.libevreactor import LibevConnection have_libev = True supported_reactors.append(LibevConnection) -except ImportError as exc: +except cassandra.DependencyException as exc: pass have_asyncio = False diff --git a/cassandra/__init__.py b/cassandra/__init__.py index 49c50ef844..9c3898298a 100644 --- a/cassandra/__init__.py +++ b/cassandra/__init__.py @@ -23,7 +23,7 @@ def emit(self, record): logging.getLogger('cassandra').addHandler(NullHandler()) -__version_info__ = (3, 28, 2) +__version_info__ = (3, 29, 0) __version__ = '.'.join(map(str, __version_info__)) @@ -735,6 +735,7 @@ class OperationType(Enum): Read = 0 Write = 1 + class RateLimitReached(ConfigurationException): ''' Rate limit was exceeded for a partition affected by the request. @@ -747,3 +748,26 @@ def __init__(self, op_type=None, rejected_by_coordinator=False): self.rejected_by_coordinator = rejected_by_coordinator message = f"[request_error_rate_limit_reached OpType={op_type.name} RejectedByCoordinator={rejected_by_coordinator}]" Exception.__init__(self, message) + + +class DependencyException(Exception): + """ + Specific exception class for handling issues with driver dependencies + """ + + excs = [] + """ + A sequence of child exceptions + """ + + def __init__(self, msg, excs=[]): + complete_msg = msg + if excs: + complete_msg += ("\nThe following exceptions were observed: \n - " + '\n - '.join(str(e) for e in excs)) + Exception.__init__(self, complete_msg) + +class VectorDeserializationFailure(DriverException): + """ + The driver was unable to deserialize a given vector + """ + pass diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 14c3074e85..55a0d9f44a 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -25,7 +25,7 @@ from collections.abc import Mapping from concurrent.futures import ThreadPoolExecutor, FIRST_COMPLETED, wait as wait_futures from copy import copy -from functools import partial, wraps +from functools import partial, reduce, wraps from itertools import groupby, count, chain import json import logging @@ -45,7 +45,7 @@ from cassandra import (ConsistencyLevel, AuthenticationFailed, InvalidRequest, OperationTimedOut, UnsupportedOperation, SchemaTargetType, DriverException, ProtocolVersion, - UnresolvableContactPoints) + UnresolvableContactPoints, DependencyException) from cassandra.auth import _proxy_execute_key, PlainTextAuthProvider from cassandra.connection import (ConnectionException, ConnectionShutdown, ConnectionHeartbeat, ProtocolVersionUnsupported, @@ -113,6 +113,19 @@ except ImportError: from cassandra.util import WeakSet # NOQA +def _is_gevent_monkey_patched(): + if 'gevent.monkey' not in sys.modules: + return False + import gevent.socket + return socket.socket is gevent.socket.socket + +def _try_gevent_import(): + if _is_gevent_monkey_patched(): + from cassandra.io.geventreactor import GeventConnection + return (GeventConnection,None) + else: + return (None,None) + def _is_eventlet_monkey_patched(): if 'eventlet.patcher' not in sys.modules: return False @@ -124,28 +137,46 @@ def _is_eventlet_monkey_patched(): # TODO: remove it when eventlet issue would be fixed return False -def _is_gevent_monkey_patched(): - if 'gevent.monkey' not in sys.modules: - return False - import gevent.socket - return socket.socket is gevent.socket.socket +def _try_eventlet_import(): + if _is_eventlet_monkey_patched(): + from cassandra.io.eventletreactor import EventletConnection + return (EventletConnection,None) + else: + return (None,None) +def _try_libev_import(): + try: + from cassandra.io.libevreactor import LibevConnection + return (LibevConnection,None) + except DependencyException as e: + return (None, e) -# default to gevent when we are monkey patched with gevent, eventlet when -# monkey patched with eventlet, otherwise if libev is available, use that as -# the default because it's fastest. Otherwise, use asyncore. -if _is_gevent_monkey_patched(): - from cassandra.io.geventreactor import GeventConnection as DefaultConnection -elif _is_eventlet_monkey_patched(): - from cassandra.io.eventletreactor import EventletConnection as DefaultConnection -else: +def _try_asyncore_import(): try: - from cassandra.io.libevreactor import LibevConnection as DefaultConnection # NOQA - except ImportError: - try: - from cassandra.io.asyncorereactor import AsyncoreConnection as DefaultConnection # NOQA - except ImportError: - from cassandra.io.asyncioreactor import AsyncioConnection as DefaultConnection # NOQA + from cassandra.io.asyncorereactor import AsyncoreConnection + return (AsyncoreConnection,None) + except DependencyException as e: + return (None, e) + +def _try_asyncio_import(): + from cassandra.io.asyncioreactor import AsyncioConnection + return (AsyncioConnection, None) + +def _connection_reduce_fn(val,import_fn): + (rv, excs) = val + # If we've already found a workable Connection class return immediately + if rv: + return val + (import_result, exc) = import_fn() + if exc: + excs.append(exc) + return (rv or import_result, excs) + +conn_fns = (_try_gevent_import, _try_eventlet_import, _try_libev_import, _try_asyncore_import, _try_asyncio_import) +(conn_class, excs) = reduce(_connection_reduce_fn, conn_fns, (None,[])) +if not conn_class: + raise DependencyException("Exception loading connection class dependencies", excs) +DefaultConnection = conn_class # Forces load of utf8 encoding module to avoid deadlock that occurs # if code that is being imported tries to import the module in a seperate @@ -802,9 +833,9 @@ def default_retry_policy(self, policy): Using ssl_options without ssl_context is deprecated and will be removed in the next major release. - An optional dict which will be used as kwargs for ``ssl.SSLContext.wrap_socket`` (or - ``ssl.wrap_socket()`` if used without ssl_context) when new sockets are created. - This should be used when client encryption is enabled in Cassandra. + An optional dict which will be used as kwargs for ``ssl.SSLContext.wrap_socket`` + when new sockets are created. This should be used when client encryption is enabled + in Cassandra. The following documentation only applies when ssl_options is used without ssl_context. @@ -820,6 +851,12 @@ def default_retry_policy(self, policy): should almost always require the option ``'cert_reqs': ssl.CERT_REQUIRED``. Note also that this functionality was not built into Python standard library until (2.7.9, 3.2). To enable this mechanism in earlier versions, patch ``ssl.match_hostname`` with a custom or `back-ported function `_. + + .. versionchanged:: 3.29.0 + + ``ssl.match_hostname`` has been deprecated since Python 3.7 (and removed in Python 3.12). This functionality is now implemented + via ``ssl.SSLContext.check_hostname``. All options specified above (including ``check_hostname``) should continue to behave in a + way that is consistent with prior implementations. """ ssl_context = None diff --git a/cassandra/connection.py b/cassandra/connection.py index ebdfe99993..a2540a967b 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -752,7 +752,6 @@ class Connection(object): _socket = None _socket_impl = socket - _ssl_impl = ssl _check_hostname = False _product_type = None @@ -780,7 +779,7 @@ def __init__(self, host='127.0.0.1', port=9042, authenticator=None, self.endpoint = host if isinstance(host, EndPoint) else DefaultEndPoint(host, port) self.authenticator = authenticator - self.ssl_options = ssl_options.copy() if ssl_options else None + self.ssl_options = ssl_options.copy() if ssl_options else {} self.ssl_context = ssl_context self.sockopts = sockopts self.compression = compression @@ -800,15 +799,20 @@ def __init__(self, host='127.0.0.1', port=9042, authenticator=None, self._on_orphaned_stream_released = on_orphaned_stream_released if ssl_options: - self._check_hostname = bool(self.ssl_options.pop('check_hostname', False)) - if self._check_hostname: - if not getattr(ssl, 'match_hostname', None): - raise RuntimeError("ssl_options specify 'check_hostname', but ssl.match_hostname is not provided. " - "Patch or upgrade Python to use this option.") self.ssl_options.update(self.endpoint.ssl_options or {}) elif self.endpoint.ssl_options: self.ssl_options = self.endpoint.ssl_options + # PYTHON-1331 + # + # We always use SSLContext.wrap_socket() now but legacy configs may have other params that were passed to ssl.wrap_socket()... + # and either could have 'check_hostname'. Remove these params into a separate map and use them to build an SSLContext if + # we need to do so. + # + # Note the use of pop() here; we are very deliberately removing these params from ssl_options if they're present. After this + # operation ssl_options should contain only args needed for the ssl_context.wrap_socket() call. + if not self.ssl_context and self.ssl_options: + self.ssl_context = self._build_ssl_context_from_options() if protocol_version >= 3: self.max_request_id = min(self.max_in_flight - 1, (2 ** 15) - 1) @@ -882,15 +886,48 @@ def factory(cls, endpoint, timeout, host_conn = None, *args, **kwargs): else: return conn + def _build_ssl_context_from_options(self): + + # Extract a subset of names from self.ssl_options which apply to SSLContext creation + ssl_context_opt_names = ['ssl_version', 'cert_reqs', 'check_hostname', 'keyfile', 'certfile', 'ca_certs', 'ciphers'] + opts = {k:self.ssl_options.get(k, None) for k in ssl_context_opt_names if k in self.ssl_options} + + # Python >= 3.10 requires either PROTOCOL_TLS_CLIENT or PROTOCOL_TLS_SERVER so we'll get ahead of things by always + # being explicit + ssl_version = opts.get('ssl_version', None) or ssl.PROTOCOL_TLS_CLIENT + cert_reqs = opts.get('cert_reqs', None) or ssl.CERT_REQUIRED + rv = ssl.SSLContext(protocol=int(ssl_version)) + rv.check_hostname = bool(opts.get('check_hostname', False)) + rv.options = int(cert_reqs) + + certfile = opts.get('certfile', None) + keyfile = opts.get('keyfile', None) + if certfile: + rv.load_cert_chain(certfile, keyfile) + ca_certs = opts.get('ca_certs', None) + if ca_certs: + rv.load_verify_locations(ca_certs) + ciphers = opts.get('ciphers', None) + if ciphers: + rv.set_ciphers(ciphers) + + return rv + def _wrap_socket_from_context(self): - ssl_options = self.ssl_options or {} + + # Extract a subset of names from self.ssl_options which apply to SSLContext.wrap_socket (or at least the parts + # of it that don't involve building an SSLContext under the covers) + wrap_socket_opt_names = ['server_side', 'do_handshake_on_connect', 'suppress_ragged_eofs', 'server_hostname'] + opts = {k:self.ssl_options.get(k, None) for k in wrap_socket_opt_names if k in self.ssl_options} + # PYTHON-1186: set the server_hostname only if the SSLContext has # check_hostname enabled and it is not already provided by the EndPoint ssl options - if (self.ssl_context.check_hostname and - 'server_hostname' not in ssl_options): - ssl_options = ssl_options.copy() - ssl_options['server_hostname'] = self.endpoint.address - self._socket = self.ssl_context.wrap_socket(self._socket, **ssl_options) + #opts['server_hostname'] = self.endpoint.address + if (self.ssl_context.check_hostname and 'server_hostname' not in opts): + server_hostname = self.endpoint.address + opts['server_hostname'] = server_hostname + + return self.ssl_context.wrap_socket(self._socket, **opts) def _initiate_connection(self, sockaddr): if self.features.shard_id is not None: @@ -904,8 +941,11 @@ def _initiate_connection(self, sockaddr): self._socket.connect(sockaddr) - def _match_hostname(self): - ssl.match_hostname(self._socket.getpeercert(), self.endpoint.address) + # PYTHON-1331 + # + # Allow implementations specific to an event loop to add additional behaviours + def _validate_hostname(self): + pass def _get_socket_addresses(self): address, port = self.endpoint.resolve() @@ -927,18 +967,21 @@ def _connect_socket(self): try: self._socket = self._socket_impl.socket(af, socktype, proto) if self.ssl_context: - self._wrap_socket_from_context() - elif self.ssl_options: - if not self._ssl_impl: - raise RuntimeError("This version of Python was not compiled with SSL support") - self._socket = self._ssl_impl.wrap_socket(self._socket, **self.ssl_options) + self._socket = self._wrap_socket_from_context() self._socket.settimeout(self.connect_timeout) self._initiate_connection(sockaddr) self._socket.settimeout(None) + local_addr = self._socket.getsockname() log.debug("Connection %s: '%s' -> '%s'", id(self), local_addr, sockaddr) + + # PYTHON-1331 + # + # Most checking is done via the check_hostname param on the SSLContext. + # Subclasses can add additional behaviours via _validate_hostname() so + # run that here. if self._check_hostname: - self._match_hostname() + self._validate_hostname() sockerr = None break except socket.error as err: diff --git a/cassandra/cqltypes.py b/cassandra/cqltypes.py index 8529897958..60b6c5ba5e 100644 --- a/cassandra/cqltypes.py +++ b/cassandra/cqltypes.py @@ -49,7 +49,7 @@ float_pack, float_unpack, double_pack, double_unpack, varint_pack, varint_unpack, point_be, point_le, vints_pack, vints_unpack) -from cassandra import util +from cassandra import util, VectorDeserializationFailure _little_endian_flag = 1 # we always serialize LE import ipaddress @@ -460,6 +460,7 @@ def serialize(uuid, protocol_version): class BooleanType(_CassandraType): typename = 'boolean' + serial_size = 1 @staticmethod def deserialize(byts, protocol_version): @@ -499,6 +500,7 @@ def serialize(var, protocol_version): class FloatType(_CassandraType): typename = 'float' + serial_size = 4 @staticmethod def deserialize(byts, protocol_version): @@ -511,6 +513,7 @@ def serialize(byts, protocol_version): class DoubleType(_CassandraType): typename = 'double' + serial_size = 8 @staticmethod def deserialize(byts, protocol_version): @@ -523,6 +526,7 @@ def serialize(byts, protocol_version): class LongType(_CassandraType): typename = 'bigint' + serial_size = 8 @staticmethod def deserialize(byts, protocol_version): @@ -535,6 +539,7 @@ def serialize(byts, protocol_version): class Int32Type(_CassandraType): typename = 'int' + serial_size = 4 @staticmethod def deserialize(byts, protocol_version): @@ -647,6 +652,7 @@ class TimestampType(DateType): class TimeUUIDType(DateType): typename = 'timeuuid' + serial_size = 16 def my_timestamp(self): return util.unix_time_from_uuid1(self.val) @@ -693,6 +699,7 @@ def serialize(val, protocol_version): class ShortType(_CassandraType): typename = 'smallint' + serial_size = 2 @staticmethod def deserialize(byts, protocol_version): @@ -705,6 +712,7 @@ def serialize(byts, protocol_version): class TimeType(_CassandraType): typename = 'time' + serial_size = 8 @staticmethod def deserialize(byts, protocol_version): @@ -1419,8 +1427,11 @@ def apply_parameters(cls, params, names): @classmethod def deserialize(cls, byts, protocol_version): - indexes = (4 * x for x in range(0, cls.vector_size)) - return [cls.subtype.deserialize(byts[idx:idx + 4], protocol_version) for idx in indexes] + serialized_size = getattr(cls.subtype, "serial_size", None) + if not serialized_size: + raise VectorDeserializationFailure("Cannot determine serialized size for vector with subtype %s" % cls.subtype.__name__) + indexes = (serialized_size * x for x in range(0, cls.vector_size)) + return [cls.subtype.deserialize(byts[idx:idx + serialized_size], protocol_version) for idx in indexes] @classmethod def serialize(cls, v, protocol_version): diff --git a/cassandra/io/asyncorereactor.py b/cassandra/io/asyncorereactor.py index c62d7fa70e..4847fed5e0 100644 --- a/cassandra/io/asyncorereactor.py +++ b/cassandra/io/asyncorereactor.py @@ -30,7 +30,15 @@ except ImportError: from cassandra.util import WeakSet # noqa -import asyncore +from cassandra import DependencyException +try: + import asyncore +except ModuleNotFoundError: + raise DependencyException( + "Unable to import asyncore module. Note that this module has been removed in Python 3.12 " + "so when using the driver with this version (or anything newer) you will need to use one of the " + "other event loop implementations." + ) from cassandra.connection import Connection, ConnectionShutdown, NONBLOCKING, Timer, TimerManager diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index 02a374cc91..7495545eb9 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -21,13 +21,13 @@ from threading import Lock, Thread import time - +from cassandra import DependencyException from cassandra.connection import (Connection, ConnectionShutdown, NONBLOCKING, Timer, TimerManager) try: import cassandra.io.libevwrapper as libev except ImportError: - raise ImportError( + raise DependencyException( "The C extension needed to use libev was not found. This " "probably means that you didn't have the required build dependencies " "when installing the driver. See " diff --git a/docs/index.rst b/docs/index.rst index ccbd8437ab..16712f0bf1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,7 +4,7 @@ A Python client driver for `Scylla `_. This driver works exclusively with the Cassandra Query Language v3 (CQL3) and Cassandra's native protocol. -The driver supports Python 3.6-3.11. +The driver supports Python 3.6-3.12. This driver is open source under the `Apache v2 License `_. diff --git a/docs/installation.rst b/docs/installation.rst index db70ce4be6..866741283d 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -3,7 +3,7 @@ Installation Supported Platforms ------------------- -Python versions 3.6-3.11 are supported. Both CPython (the standard Python +Python versions 3.6-3.12 are supported. Both CPython (the standard Python implementation) and `PyPy `_ are supported and tested. Linux, OSX, and Windows are supported. @@ -26,7 +26,7 @@ To check if the installation was successful, you can run:: python -c 'import cassandra; print cassandra.__version__' -It should print something like "3.27.0". +It should print something like "3.29.0". (*Optional*) Compression Support -------------------------------- @@ -182,12 +182,15 @@ dependencies, then use install-option:: sudo pip install --install-option="--no-cython" +Supported Event Loops +^^^^^^^^^^^^^^^^^^^^^ +For Python versions before 3.12 the driver uses the ``asyncore`` module for its default +event loop. Other event loops such as ``libev``, ``gevent`` and ``eventlet`` are also +available via Python modules or C extensions. Python 3.12 has removed ``asyncore`` entirely +so for this platform one of these other event loops must be used. + libev support ^^^^^^^^^^^^^ -The driver currently uses Python's ``asyncore`` module for its default -event loop. For better performance, ``libev`` is also supported through -a C extension. - If you're on Linux, you should be able to install libev through a package manager. For example, on Debian/Ubuntu:: diff --git a/pyproject.toml b/pyproject.toml index 6647b6b65c..31d9cb888f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,8 +12,6 @@ classifiers = [ 'Natural Language :: English', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', diff --git a/test-requirements.txt b/test-requirements.txt index 2851efc3db..4a40e69ad9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,13 +5,12 @@ mock>1.1 pytz sure pure-sasl -twisted[tls]; python_version >= '3.5' -twisted[tls]==19.2.1; python_version < '3.5' -gevent>=1.0; python_version < '3.13' and platform_machine != 'i686' and platform_machine != 'win32' -gevent==23.9.0; python_version < '3.13' and (platform_machine == 'i686' or platform_machine == 'win32') -eventlet>=0.33.3; python_version < '3.13' -cython +twisted[tls] +gevent>=1.0; platform_machine != 'i686' and platform_machine != 'win32' +gevent==23.9.0; platform_machine == 'i686' or platform_machine == 'win32' +eventlet>=0.33.3; +cython>=0.20,<0.30 packaging -futurist; python_version >= '3.7' -asynctest; python_version >= '3.5' +futurist +asynctest pyyaml diff --git a/tests/__init__.py b/tests/__init__.py index c55a6567e4..56a9759445 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -20,6 +20,8 @@ import os from concurrent.futures import ThreadPoolExecutor +from cassandra import DependencyException + log = logging.getLogger() def is_eventlet_monkey_patched(): @@ -27,10 +29,10 @@ def is_eventlet_monkey_patched(): return False try: import eventlet.patcher + return eventlet.patcher.is_monkey_patched('socket') + # Yet another case related to PYTHON-1364 except AttributeError: return False - return eventlet.patcher.is_monkey_patched('socket') - def is_gevent_monkey_patched(): if 'gevent.monkey' not in sys.modules: @@ -88,17 +90,18 @@ def is_monkey_patched(): elif "asyncio" in EVENT_LOOP_MANAGER: from cassandra.io.asyncioreactor import AsyncioConnection connection_class = AsyncioConnection - else: + log.debug("Using default event loop (libev)") try: from cassandra.io.libevreactor import LibevConnection connection_class = LibevConnection - except ImportError as e: + except DependencyException as e: log.debug('Could not import LibevConnection, ' 'using connection_class=None; ' 'failed with error:\n {}'.format( repr(e) )) + log.debug("Will attempt to set connection class at cluster initialization") connection_class = None diff --git a/tests/integration/long/test_ipv6.py b/tests/integration/long/test_ipv6.py index f4a245f704..d58bad987a 100644 --- a/tests/integration/long/test_ipv6.py +++ b/tests/integration/long/test_ipv6.py @@ -15,11 +15,12 @@ import os, socket, errno from ccmlib import common +from cassandra import DependencyException from cassandra.cluster import NoHostAvailable try: from cassandra.io.asyncorereactor import AsyncoreConnection -except ImportError: +except DependencyException: AsyncoreConnection = None from tests import is_monkey_patched @@ -27,7 +28,7 @@ try: from cassandra.io.libevreactor import LibevConnection -except ImportError: +except DependencyException: LibevConnection = None diff --git a/tests/integration/long/test_ssl.py b/tests/integration/long/test_ssl.py index f868351fe5..070e2fe268 100644 --- a/tests/integration/long/test_ssl.py +++ b/tests/integration/long/test_ssl.py @@ -28,7 +28,7 @@ if not hasattr(ssl, 'match_hostname'): try: - from backports.ssl_match_hostname import match_hostname + from ssl import match_hostname ssl.match_hostname = match_hostname except ImportError: pass # tests will fail diff --git a/tests/integration/standard/test_connection.py b/tests/integration/standard/test_connection.py index 40016b5a04..6329b0b293 100644 --- a/tests/integration/standard/test_connection.py +++ b/tests/integration/standard/test_connection.py @@ -23,16 +23,9 @@ import time from unittest import SkipTest -from cassandra import ConsistencyLevel, OperationTimedOut +from cassandra import ConsistencyLevel, OperationTimedOut, DependencyException from cassandra.cluster import NoHostAvailable, ConnectionShutdown, ExecutionProfile, EXEC_PROFILE_DEFAULT - -try: - from cassandra.io.asyncorereactor import AsyncoreConnection -except ImportError: - AsyncoreConnection = None - from cassandra.protocol import QueryMessage -from cassandra.connection import Connection from cassandra.policies import HostFilterPolicy, RoundRobinPolicy, HostStateListener from cassandra.pool import HostConnectionPool @@ -40,10 +33,16 @@ from tests.integration import use_singledc, get_node, CASSANDRA_IP, local, \ requiresmallclockgranularity, greaterthancass20, TestCluster +try: + import cassandra.io.asyncorereactor + from cassandra.io.asyncorereactor import AsyncoreConnection +except DependencyException: + AsyncoreConnection = None + try: from cassandra.io.libevreactor import LibevConnection import cassandra.io.libevreactor -except ImportError: +except DependencyException: LibevConnection = None @@ -447,8 +446,7 @@ def setUp(self): if is_monkey_patched(): raise unittest.SkipTest("Can't test asyncore with monkey patching") if AsyncoreConnection is None: - raise unittest.SkipTest( - 'asyncore does not appear to be installed properly') + raise unittest.SkipTest('Unable to import asyncore module') ConnectionTests.setUp(self) def clean_global_loop(self): diff --git a/tests/integration/standard/test_scylla_cloud.py b/tests/integration/standard/test_scylla_cloud.py index 80af88d731..5679d959bb 100644 --- a/tests/integration/standard/test_scylla_cloud.py +++ b/tests/integration/standard/test_scylla_cloud.py @@ -5,6 +5,7 @@ from ccmlib.utils.ssl_utils import generate_ssl_stores from ccmlib.utils.sni_proxy import refresh_certs, start_sni_proxy, create_cloud_config, NodeInfo +from cassandra import DependencyException from cassandra.policies import TokenAwarePolicy, RoundRobinPolicy, ConstantReconnectionPolicy from tests.integration import use_cluster, PROTOCOL_VERSION from cassandra.cluster import Cluster, TwistedConnection @@ -15,14 +16,14 @@ try: from cassandra.io.libevreactor import LibevConnection supported_connection_classes += [LibevConnection] -except ImportError: +except DependencyException: pass try: from cassandra.io.asyncorereactor import AsyncoreConnection supported_connection_classes += [AsyncoreConnection] -except ImportError: +except DependencyException: pass #from cassandra.io.geventreactor import GeventConnection diff --git a/tests/unit/io/test_asyncorereactor.py b/tests/unit/io/test_asyncorereactor.py index e9fe9aa2cb..1941d0c839 100644 --- a/tests/unit/io/test_asyncorereactor.py +++ b/tests/unit/io/test_asyncorereactor.py @@ -15,11 +15,14 @@ from mock import patch import socket + +from cassandra import DependencyException + try: import cassandra.io.asyncorereactor as asyncorereactor from cassandra.io.asyncorereactor import AsyncoreConnection ASYNCCORE_AVAILABLE = True -except ImportError: +except (ImportError, DependencyException): ASYNCCORE_AVAILABLE = False AsyncoreConnection = None diff --git a/tests/unit/io/test_libevreactor.py b/tests/unit/io/test_libevreactor.py index 67ab5fc7d6..e2c04fdf8e 100644 --- a/tests/unit/io/test_libevreactor.py +++ b/tests/unit/io/test_libevreactor.py @@ -19,12 +19,12 @@ from tests import is_monkey_patched from tests.unit.io.utils import ReactorTestMixin, TimerTestMixin, noop_if_monkey_patched - +from cassandra import DependencyException try: from cassandra.io.libevreactor import _cleanup as libev__cleanup from cassandra.io.libevreactor import LibevConnection -except ImportError: +except (ImportError, DependencyException): LibevConnection = None # noqa diff --git a/tests/unit/test_cluster.py b/tests/unit/test_cluster.py index 3334e650a5..24cd4047d1 100644 --- a/tests/unit/test_cluster.py +++ b/tests/unit/test_cluster.py @@ -14,6 +14,7 @@ import unittest import logging +import socket from mock import patch, Mock @@ -88,8 +89,9 @@ class ClusterTest(unittest.TestCase): def test_tuple_for_contact_points(self): cluster = Cluster(contact_points=[('localhost', 9045), ('127.0.0.2', 9046), '127.0.0.3'], port=9999) + localhost_addr = set([addr[0] for addr in [t for (_,_,_,_,t) in socket.getaddrinfo("localhost",80)]]) for cp in cluster.endpoints_resolved: - if cp.address in ('::1', '127.0.0.1'): + if cp.address in localhost_addr: self.assertEqual(cp.port, 9045) elif cp.address == '127.0.0.2': self.assertEqual(cp.port, 9046) diff --git a/tests/unit/test_types.py b/tests/unit/test_types.py index a06bbd452d..5db7f087b7 100644 --- a/tests/unit/test_types.py +++ b/tests/unit/test_types.py @@ -16,10 +16,11 @@ import datetime import tempfile import time +import uuid from binascii import unhexlify import cassandra -from cassandra import util +from cassandra import util, VectorDeserializationFailure from cassandra.cqltypes import ( CassandraType, DateRangeType, DateType, DecimalType, EmptyValue, LongType, SetType, UTF8Type, @@ -308,15 +309,67 @@ def test_cql_quote(self): self.assertEqual(cql_quote('test'), "'test'") self.assertEqual(cql_quote(0), '0') - def test_vector_round_trip(self): - base = [3.4, 2.9, 41.6, 12.0] - ctype = parse_casstype_args("org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.FloatType, 4)") - base_bytes = ctype.serialize(base, 0) - self.assertEqual(16, len(base_bytes)) - result = ctype.deserialize(base_bytes, 0) - self.assertEqual(len(base), len(result)) - for idx in range(0,len(base)): - self.assertAlmostEqual(base[idx], result[idx], places=5) + def test_vector_round_trip_types_with_serialized_size(self): + # Test all the types which specify a serialized size... see PYTHON-1371 for details + self._round_trip_test([True, False, False, True], \ + "org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.BooleanType, 4)") + self._round_trip_test([3.4, 2.9, 41.6, 12.0], \ + "org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.FloatType, 4)") + self._round_trip_test([3.4, 2.9, 41.6, 12.0], \ + "org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.DoubleType, 4)") + self._round_trip_test([3, 2, 41, 12], \ + "org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.LongType, 4)") + self._round_trip_test([3, 2, 41, 12], \ + "org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.Int32Type, 4)") + self._round_trip_test([uuid.uuid1(), uuid.uuid1(), uuid.uuid1(), uuid.uuid1()], \ + "org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.TimeUUIDType, 4)") + self._round_trip_test([3, 2, 41, 12], \ + "org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.ShortType, 4)") + self._round_trip_test([datetime.time(1,1,1), datetime.time(2,2,2), datetime.time(3,3,3)], \ + "org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.TimeType, 3)") + + def test_vector_round_trip_types_without_serialized_size(self): + # Test all the types which do not specify a serialized size... see PYTHON-1371 for details + # Varints + with self.assertRaises(VectorDeserializationFailure): + self._round_trip_test([3, 2, 41, 12], \ + "org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.IntegerType, 4)") + # ASCII text + with self.assertRaises(VectorDeserializationFailure): + self._round_trip_test(["abc", "def", "ghi", "jkl"], \ + "org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.AsciiType, 4)") + # UTF8 text + with self.assertRaises(VectorDeserializationFailure): + self._round_trip_test(["abc", "def", "ghi", "jkl"], \ + "org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.UTF8Type, 4)") + # Duration (containts varints) + with self.assertRaises(VectorDeserializationFailure): + self._round_trip_test([util.Duration(1,1,1), util.Duration(2,2,2), util.Duration(3,3,3)], \ + "org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.DurationType, 3)") + # List (of otherwise serializable type) + with self.assertRaises(VectorDeserializationFailure): + self._round_trip_test([[3.4], [2.9], [41.6], [12.0]], \ + "org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.ListType(org.apache.cassandra.db.marshal.FloatType), 4)") + # Set (of otherwise serializable type) + with self.assertRaises(VectorDeserializationFailure): + self._round_trip_test([set([3.4]), set([2.9]), set([41.6]), set([12.0])], \ + "org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.SetType(org.apache.cassandra.db.marshal.FloatType), 4)") + # Map (of otherwise serializable types) + with self.assertRaises(VectorDeserializationFailure): + self._round_trip_test([{1:3.4}, {2:2.9}, {3:41.6}, {4:12.0}], \ + "org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.MapType \ + (org.apache.cassandra.db.marshal.Int32Type,org.apache.cassandra.db.marshal.FloatType), 4)") + + def _round_trip_test(self, data, ctype_str): + ctype = parse_casstype_args(ctype_str) + data_bytes = ctype.serialize(data, 0) + serialized_size = getattr(ctype.subtype, "serial_size", None) + if serialized_size: + self.assertEqual(serialized_size * len(data), len(data_bytes)) + result = ctype.deserialize(data_bytes, 0) + self.assertEqual(len(data), len(result)) + for idx in range(0,len(data)): + self.assertAlmostEqual(data[idx], result[idx], places=5) def test_vector_cql_parameterized_type(self): ctype = parse_casstype_args("org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.FloatType, 4)")