From 0a05f1069883a1eaf22c25d5a33809308857a9ca Mon Sep 17 00:00:00 2001 From: Zhen Date: Tue, 3 Oct 2017 11:21:28 +0200 Subject: [PATCH 1/4] Add examples for driver documentation --- .../config_connection_pool_example.py | 39 +++++++++++++++++++ .../config_connection_timeout_example.py | 4 ++ .../config_load_balancing_strategy_example.py | 37 ++++++++++++++++++ .../examples/config_max_retry_time_example.py | 4 ++ test/examples/test_examples.py | 20 ++++++++++ 5 files changed, 104 insertions(+) create mode 100644 test/examples/config_connection_pool_example.py create mode 100644 test/examples/config_load_balancing_strategy_example.py diff --git a/test/examples/config_connection_pool_example.py b/test/examples/config_connection_pool_example.py new file mode 100644 index 00000000..14622b19 --- /dev/null +++ b/test/examples/config_connection_pool_example.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +# Copyright (c) 2002-2017 "Neo Technology," +# Network Engine for Objects in Lund AB [http://neotechnology.com] +# +# This file is part of Neo4j. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# tag::config-connection-pool-import[] +from neo4j.v1 import GraphDatabase +# end::config-connection-pool-import[] + + +class ConfigConnectionPoolExample: + # tag::config-connection-pool[] + def __init__(self, uri, user, password): + self._driver = GraphDatabase.driver(uri, auth=(user, password), + max_connection_lifetime=30 * 60, max_connection_pool_size=50, + connection_acquisition_timeout=2 * 60) + # end::config-connection-pool[] + + def close(self): + self._driver.close() + + def can_connect(driver): + result = driver.session().run("RETURN 1") + return result.single()[0] == 1 \ No newline at end of file diff --git a/test/examples/config_connection_timeout_example.py b/test/examples/config_connection_timeout_example.py index 1360507e..8eadf33b 100644 --- a/test/examples/config_connection_timeout_example.py +++ b/test/examples/config_connection_timeout_example.py @@ -31,3 +31,7 @@ def __init__(self, uri, user, password): def close(self): self._driver.close() + + def can_connect(self): + result = self._driver.session().run("RETURN 1") + return result.single()[0] == 1 diff --git a/test/examples/config_load_balancing_strategy_example.py b/test/examples/config_load_balancing_strategy_example.py new file mode 100644 index 00000000..f35c826c --- /dev/null +++ b/test/examples/config_load_balancing_strategy_example.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +# Copyright (c) 2002-2017 "Neo Technology," +# Network Engine for Objects in Lund AB [http://neotechnology.com] +# +# This file is part of Neo4j. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# tag::config-load-balancing-strategy-import[] +from neo4j.v1 import GraphDatabase, LOAD_BALANCING_STRATEGY_LEAST_CONNECTED +# end::config-load-balancing-strategy-import[] + + +class ConfigLoadBalancingStrategyExample: + # tag::config-load-balancing-strategy[] + def __init__(self, uri, user, password): + self._driver = GraphDatabase.driver(uri, auth=(user, password), load_balancing_strategy=LOAD_BALANCING_STRATEGY_LEAST_CONNECTED) + # end::config-load-balancing-strategy[] + + def close(self): + self._driver.close() + + def can_connect(self): + result = self._driver.session().run("RETURN 1") + return result.single()[0] == 1 diff --git a/test/examples/config_max_retry_time_example.py b/test/examples/config_max_retry_time_example.py index 59304c43..0df66455 100644 --- a/test/examples/config_max_retry_time_example.py +++ b/test/examples/config_max_retry_time_example.py @@ -31,3 +31,7 @@ def __init__(self, uri, user, password): def close(self): self._driver.close() + + def can_connect(self): + result = self._driver.session().run("RETURN 1") + return result.single()[0] == 1 diff --git a/test/examples/test_examples.py b/test/examples/test_examples.py index cad93db1..9549766c 100644 --- a/test/examples/test_examples.py +++ b/test/examples/test_examples.py @@ -52,6 +52,26 @@ def test_autocommit_transaction_example(self): self.assertTrue(self.person_count('Alice') > 0) + def test_config_connection_pool_example(self): + from test.examples.config_connection_pool_example import ConfigConnectionPoolExample + example = ConfigConnectionPoolExample(self.bolt_uri, self.user, self.password) + self.assertTrue(example.can_connect()) + + def test_connection_timeout_example(self): + from test.examples.config_connection_timeout_example import ConfigConnectionTimeoutExample + example = ConfigConnectionTimeoutExample(self.bolt_uri, self.user, self.password) + self.assertTrue(example.can_connect()) + + def test_load_balancing_strategy_example(self): + from test.examples.config_load_balancing_strategy_example import ConfigLoadBalancingStrategyExample + example = ConfigLoadBalancingStrategyExample(self.bolt_uri, self.user, self.password) + self.assertTrue(example.can_connect()) + + def test_max_retry_time_example(self): + from test.examples.config_max_retry_time_example import ConfigMaxRetryTimeExample + example = ConfigMaxRetryTimeExample(self.bolt_uri, self.user, self.password) + self.assertTrue(example.can_connect()) + def test_basic_auth_example(self): from test.examples.auth_example import BasicAuthExample From ab43b4b18dda9efede6fd94f7771a0bc079fe805 Mon Sep 17 00:00:00 2001 From: Zhen Date: Tue, 3 Oct 2017 17:00:53 +0200 Subject: [PATCH 2/4] Default max connection pool size to 250 --- neo4j/bolt/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo4j/bolt/connection.py b/neo4j/bolt/connection.py index 5cf1d6d4..3b0f2e47 100644 --- a/neo4j/bolt/connection.py +++ b/neo4j/bolt/connection.py @@ -50,7 +50,7 @@ INFINITE = -1 DEFAULT_MAX_CONNECTION_LIFETIME = INFINITE -DEFAULT_MAX_CONNECTION_POOL_SIZE = INFINITE +DEFAULT_MAX_CONNECTION_POOL_SIZE = 250 DEFAULT_CONNECTION_TIMEOUT = 5.0 DEFAULT_CONNECTION_ACQUISITION_TIMEOUT = 60 DEFAULT_PORT = 7687 From 8c6c1d02452c85ae6c8a2405b90b50903795956f Mon Sep 17 00:00:00 2001 From: Zhen Date: Wed, 4 Oct 2017 14:43:15 +0200 Subject: [PATCH 3/4] Extract all config options and default values into a default_config dict for better reference inside the code --- neo4j/bolt/__init__.py | 2 +- neo4j/bolt/connection.py | 25 +++++--------- neo4j/config.py | 70 ++++++++++++++++++++++++++++++++++++++++ neo4j/v1/__init__.py | 2 +- neo4j/v1/api.py | 9 +++--- neo4j/v1/routing.py | 7 ++-- neo4j/v1/security.py | 13 ++------ 7 files changed, 90 insertions(+), 38 deletions(-) create mode 100644 neo4j/config.py diff --git a/neo4j/bolt/__init__.py b/neo4j/bolt/__init__.py index aba36af7..04860cbd 100644 --- a/neo4j/bolt/__init__.py +++ b/neo4j/bolt/__init__.py @@ -21,4 +21,4 @@ from .cert import * from .connection import * -from .response import * +from .response import * \ No newline at end of file diff --git a/neo4j/bolt/connection.py b/neo4j/bolt/connection.py index 3b0f2e47..93a43964 100644 --- a/neo4j/bolt/connection.py +++ b/neo4j/bolt/connection.py @@ -39,23 +39,15 @@ from neo4j.bolt.response import InitResponse, AckFailureResponse, ResetResponse from neo4j.compat.ssl import SSL_AVAILABLE, HAS_SNI, SSLError from neo4j.exceptions import ClientError, ProtocolError, SecurityError, ServiceUnavailable -from neo4j.meta import version from neo4j.packstream import Packer, Unpacker from neo4j.util import import_best as _import_best from time import clock +from neo4j.config import default_config, INFINITE, TRUST_ON_FIRST_USE ChunkedInputBuffer = _import_best("neo4j.bolt._io", "neo4j.bolt.io").ChunkedInputBuffer ChunkedOutputBuffer = _import_best("neo4j.bolt._io", "neo4j.bolt.io").ChunkedOutputBuffer - -INFINITE = -1 -DEFAULT_MAX_CONNECTION_LIFETIME = INFINITE -DEFAULT_MAX_CONNECTION_POOL_SIZE = 250 -DEFAULT_CONNECTION_TIMEOUT = 5.0 -DEFAULT_CONNECTION_ACQUISITION_TIMEOUT = 60 DEFAULT_PORT = 7687 -DEFAULT_USER_AGENT = "neo4j-python/%s" % version - MAGIC_PREAMBLE = 0x6060B017 @@ -183,11 +175,11 @@ def __init__(self, address, sock, error_handler, **config): self.packer = Packer(self.output_buffer) self.unpacker = Unpacker() self.responses = deque() - self._max_connection_lifetime = config.get("max_connection_lifetime", DEFAULT_MAX_CONNECTION_LIFETIME) + self._max_connection_lifetime = config.get("max_connection_lifetime", default_config["max_connection_lifetime"]) self._creation_timestamp = clock() # Determine the user agent and ensure it is a Unicode value - user_agent = config.get("user_agent", DEFAULT_USER_AGENT) + user_agent = config.get("user_agent", default_config["user_agent"]) if isinstance(user_agent, bytes): user_agent = user_agent.decode("UTF-8") self.user_agent = user_agent @@ -413,8 +405,8 @@ def __init__(self, connector, connection_error_handler, **config): self.connections = {} self.lock = RLock() self.cond = Condition(self.lock) - self._max_connection_pool_size = config.get("max_connection_pool_size", DEFAULT_MAX_CONNECTION_POOL_SIZE) - self._connection_acquisition_timeout = config.get("connection_acquisition_timeout", DEFAULT_CONNECTION_ACQUISITION_TIMEOUT) + self._max_connection_pool_size = config.get("max_connection_pool_size", default_config["max_connection_pool_size"]) + self._connection_acquisition_timeout = config.get("connection_acquisition_timeout", default_config["connection_acquisition_timeout"]) def __enter__(self): return self @@ -546,10 +538,10 @@ def connect(address, ssl_context=None, error_handler=None, **config): else: raise ValueError("Unsupported address {!r}".format(address)) t = s.gettimeout() - s.settimeout(config.get("connection_timeout", DEFAULT_CONNECTION_TIMEOUT)) + s.settimeout(config.get("connection_timeout", default_config["connection_timeout"])) s.connect(address) s.settimeout(t) - s.setsockopt(SOL_SOCKET, SO_KEEPALIVE, 1 if config.get("keep_alive", True) else 0) + s.setsockopt(SOL_SOCKET, SO_KEEPALIVE, 1 if config.get("keep_alive", default_config["keep_alive"]) else 0) except SocketTimeout: if s: try: @@ -582,14 +574,13 @@ def connect(address, ssl_context=None, error_handler=None, **config): error.__cause__ = cause raise error else: - from neo4j.v1 import TRUST_DEFAULT, TRUST_ON_FIRST_USE # Check that the server provides a certificate der_encoded_server_certificate = s.getpeercert(binary_form=True) if der_encoded_server_certificate is None: s.close() raise ProtocolError("When using a secure socket, the server should always " "provide a certificate") - trust = config.get("trust", TRUST_DEFAULT) + trust = config.get("trust", default_config["trust"]) if trust == TRUST_ON_FIRST_USE: from neo4j.bolt.cert import PersonalCertificateStore store = PersonalCertificateStore() diff --git a/neo4j/config.py b/neo4j/config.py new file mode 100644 index 00000000..710fe859 --- /dev/null +++ b/neo4j/config.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +# Copyright (c) 2002-2017 "Neo Technology," +# Network Engine for Objects in Lund AB [http://neotechnology.com] +# +# This file is part of Neo4j. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from neo4j.meta import version + +# Auth +TRUST_ON_FIRST_USE = 0 # Deprecated +TRUST_SIGNED_CERTIFICATES = 1 # Deprecated +TRUST_ALL_CERTIFICATES = 2 +TRUST_CUSTOM_CA_SIGNED_CERTIFICATES = 3 +TRUST_SYSTEM_CA_SIGNED_CERTIFICATES = 4 +TRUST_DEFAULT = TRUST_ALL_CERTIFICATES + +# Connection Pool Management +INFINITE = -1 +DEFAULT_MAX_CONNECTION_LIFETIME = INFINITE +DEFAULT_MAX_CONNECTION_POOL_SIZE = 250 +DEFAULT_CONNECTION_TIMEOUT = 5.0 # 5s + +# Connection Settings +DEFAULT_CONNECTION_ACQUISITION_TIMEOUT = 60 # 1m + +# Routing settings +DEFAULT_MAX_RETRY_TIME = 30.0 # 30s + +LOAD_BALANCING_STRATEGY_LEAST_CONNECTED = 0 +LOAD_BALANCING_STRATEGY_ROUND_ROBIN = 1 +DEFAULT_LOAD_BALANCING_STRATEGY = LOAD_BALANCING_STRATEGY_LEAST_CONNECTED + +# Client name +DEFAULT_USER_AGENT = "neo4j-python/%s" % version + +default_config = { + "auth": None, # provide your own authentication token such as {"username", "password"} + "encrypted": True, + "trust": TRUST_DEFAULT, + "der-encoded_server_certificate": None, + + "user_agent": DEFAULT_USER_AGENT, + + # Connection pool management + "max_connection_lifetime": DEFAULT_MAX_CONNECTION_LIFETIME, + "max_connection_pool_size": DEFAULT_MAX_CONNECTION_POOL_SIZE, + "connection_acquisition_timeout": DEFAULT_CONNECTION_ACQUISITION_TIMEOUT, + + # Connection settings: + "connection_timeout": DEFAULT_CONNECTION_TIMEOUT, + "keep_alive": True, + + # Routing settings: + "max_retry_time": DEFAULT_MAX_RETRY_TIME, + "load_balancing_strategy": DEFAULT_LOAD_BALANCING_STRATEGY +} diff --git a/neo4j/v1/__init__.py b/neo4j/v1/__init__.py index a2eacc33..0e744ba8 100644 --- a/neo4j/v1/__init__.py +++ b/neo4j/v1/__init__.py @@ -23,8 +23,8 @@ from .exceptions import * from .result import * from .routing import * -from .session import * from .security import * +from .session import * from .types import * # Register supported URI schemes diff --git a/neo4j/v1/api.py b/neo4j/v1/api.py index e6d2358f..3ea5278e 100644 --- a/neo4j/v1/api.py +++ b/neo4j/v1/api.py @@ -28,6 +28,7 @@ from neo4j.exceptions import ProtocolError, ServiceUnavailable from neo4j.compat import urlparse from neo4j.exceptions import CypherError, TransientError +from neo4j.config import default_config from .exceptions import DriverError, SessionError, SessionExpired, TransactionError @@ -36,12 +37,10 @@ READ_ACCESS = "READ" WRITE_ACCESS = "WRITE" -DEFAULT_MAX_RETRY_TIME = 30.0 INITIAL_RETRY_DELAY = 1.0 RETRY_DELAY_MULTIPLIER = 2.0 RETRY_DELAY_JITTER_FACTOR = 0.2 - def last_bookmark(b0, b1): """ Return the latest of two bookmarks by looking for the maximum integer value following the last colon in the bookmark string. @@ -114,6 +113,8 @@ def driver(cls, uri, **config): `user_agent` A custom user agent string, if required. + for more config options see neo4j.config.default_config + """ parsed = urlparse(uri) try: @@ -145,7 +146,7 @@ class Driver(object): def __init__(self, pool, **config): self._lock = RLock() self._pool = pool - self._max_retry_time = config.get("max_retry_time", DEFAULT_MAX_RETRY_TIME) + self._max_retry_time = config.get("max_retry_time", default_config["max_retry_time"]) def __del__(self): self.close() @@ -233,7 +234,7 @@ class Session(object): _bookmarks = () # Default maximum time to keep retrying failed transactions. - _max_retry_time = DEFAULT_MAX_RETRY_TIME + _max_retry_time = default_config["max_retry_time"] _closed = False diff --git a/neo4j/v1/routing.py b/neo4j/v1/routing.py index 7eb845d6..4f9fb33f 100644 --- a/neo4j/v1/routing.py +++ b/neo4j/v1/routing.py @@ -31,10 +31,7 @@ from neo4j.v1.exceptions import SessionExpired from neo4j.v1.security import SecurityPlan from neo4j.v1.session import BoltSession - -LOAD_BALANCING_STRATEGY_LEAST_CONNECTED = 0 -LOAD_BALANCING_STRATEGY_ROUND_ROBIN = 1 -DEFAULT_LOAD_BALANCING_STRATEGY = LOAD_BALANCING_STRATEGY_LEAST_CONNECTED +from neo4j.config import default_config, LOAD_BALANCING_STRATEGY_LEAST_CONNECTED, LOAD_BALANCING_STRATEGY_ROUND_ROBIN class OrderedSet(MutableSet): @@ -166,7 +163,7 @@ class LoadBalancingStrategy(object): @classmethod def build(cls, connection_pool, **config): - load_balancing_strategy = config.get("load_balancing_strategy", DEFAULT_LOAD_BALANCING_STRATEGY) + load_balancing_strategy = config.get("load_balancing_strategy", default_config["load_balancing_strategy"]) if load_balancing_strategy == LOAD_BALANCING_STRATEGY_LEAST_CONNECTED: return LeastConnectedLoadBalancingStrategy(connection_pool) elif load_balancing_strategy == LOAD_BALANCING_STRATEGY_ROUND_ROBIN: diff --git a/neo4j/v1/security.py b/neo4j/v1/security.py index f94058f1..9c4145af 100644 --- a/neo4j/v1/security.py +++ b/neo4j/v1/security.py @@ -21,19 +21,12 @@ from warnings import warn from neo4j.compat.ssl import SSL_AVAILABLE, SSLContext, PROTOCOL_SSLv23, OP_NO_SSLv2, CERT_REQUIRED - +from neo4j.config import default_config, TRUST_ALL_CERTIFICATES, TRUST_CUSTOM_CA_SIGNED_CERTIFICATES, TRUST_ON_FIRST_USE, TRUST_SIGNED_CERTIFICATES, TRUST_SYSTEM_CA_SIGNED_CERTIFICATES ENCRYPTION_OFF = 0 ENCRYPTION_ON = 1 ENCRYPTION_DEFAULT = ENCRYPTION_ON if SSL_AVAILABLE else ENCRYPTION_OFF -TRUST_ON_FIRST_USE = 0 # Deprecated -TRUST_SIGNED_CERTIFICATES = 1 # Deprecated -TRUST_ALL_CERTIFICATES = 2 -TRUST_CUSTOM_CA_SIGNED_CERTIFICATES = 3 -TRUST_SYSTEM_CA_SIGNED_CERTIFICATES = 4 -TRUST_DEFAULT = TRUST_ALL_CERTIFICATES - class AuthToken(object): """ Container for auth information @@ -56,10 +49,10 @@ class SecurityPlan(object): @classmethod def build(cls, **config): - encrypted = config.get("encrypted", None) + encrypted = config.get("encrypted", default_config["encrypted"]) if encrypted is None: encrypted = _encryption_default() - trust = config.get("trust", TRUST_DEFAULT) + trust = config.get("trust", default_config["trust"]) if encrypted: if not SSL_AVAILABLE: raise RuntimeError("Bolt over TLS is only available in Python 2.7.9+ and " From cc365bd679e6f857deafa8fd3d0cffed155b7606 Mon Sep 17 00:00:00 2001 From: Zhen Date: Thu, 5 Oct 2017 15:29:26 +0200 Subject: [PATCH 4/4] Change defalut setting values --- neo4j/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neo4j/config.py b/neo4j/config.py index 710fe859..6d3d3ddb 100644 --- a/neo4j/config.py +++ b/neo4j/config.py @@ -30,8 +30,8 @@ # Connection Pool Management INFINITE = -1 -DEFAULT_MAX_CONNECTION_LIFETIME = INFINITE -DEFAULT_MAX_CONNECTION_POOL_SIZE = 250 +DEFAULT_MAX_CONNECTION_LIFETIME = 3600 # 1h +DEFAULT_MAX_CONNECTION_POOL_SIZE = 100 DEFAULT_CONNECTION_TIMEOUT = 5.0 # 5s # Connection Settings