Migrating stub tests to testkit

 - remove all migrated stub tests
 - some stub tests (or parts of them) got turned into unit tests
 - implement testkit messages ForcedRoutingTableUpdate and GetRoutingTable
 - send debug log output to testkit to easy test debugging for people running
   the backend in docker or similar

Author: robsdedude

testkitbackend/backend.py

@@ -14,19 +14,31 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from inspect import getmembers, isfunction
+from inspect import (
+    getmembers,
+    isfunction,
+)
+import io
 from json import loads, dumps
 import logging
 import sys
 import traceback
 
-from neo4j.exceptions import Neo4jError, DriverError, ServiceUnavailable
+from neo4j.exceptions import (
+    DriverError,
+    Neo4jError,
+    UnsupportedServerProduct,
+)
 
 import testkitbackend.requests as requests
 
+buffer_handler = logging.StreamHandler(io.StringIO())
+buffer_handler.setLevel(logging.DEBUG)
+
 handler = logging.StreamHandler(sys.stdout)
 handler.setLevel(logging.DEBUG)
 logging.getLogger("neo4j").addHandler(handler)
+logging.getLogger("neo4j").addHandler(buffer_handler)
 logging.getLogger("neo4j").setLevel(logging.DEBUG)
 
 log = logging.getLogger("testkitbackend")
@@ -140,7 +152,7 @@ def _process(self, request):
                     "Backend does not support some properties of the " + name +
                     " request: " + ", ".join(unsused_keys)
                 )
-        except (Neo4jError, DriverError) as e:
+        except (Neo4jError, DriverError, UnsupportedServerProduct) as e:
             log.debug(traceback.format_exc())
             if isinstance(e, Neo4jError):
                 msg = "" if e.message is None else str(e.message)
@@ -161,6 +173,14 @@ def _process(self, request):
     def send_response(self, name, data):
         """ Sends a response to backend.
         """
+        buffer_handler.acquire()
+        log_output = buffer_handler.stream.getvalue()
+        buffer_handler.stream.truncate(0)
+        buffer_handler.stream.seek(0)
+        buffer_handler.release()
+        if not log_output.endswith("\n"):
+            log_output += "\n"
+        self._wr.write(log_output.encode("utf-8"))
         response = {"name": name, "data": data}
         response = dumps(response)
         log.info(">>> " + name + dumps(data))

testkitbackend/requests.py

@@ -23,8 +23,14 @@
 from testkitbackend.fromtestkit import to_meta_and_timeout
 
 
-with open(path.join(path.dirname(__file__), "skipped_tests.json"), "r") as fd:
-    SKIPPED_TESTS = json.load(fd)
+def load_config():
+    with open(path.join(path.dirname(__file__), "test_config.json"), "r") as fd:
+        config = json.load(fd)
+    return (config["skips"],
+            [k for k, v in config["features"].items() if v is True])
+
+
+SKIPPED_TESTS, FEATURES = load_config()
 
 
 def StartTest(backend, data):
@@ -35,6 +41,10 @@ def StartTest(backend, data):
         backend.send_response("RunTest", {})
 
 
+def GetFeatures(backend, data):
+    backend.send_response("FeatureList", {"features": FEATURES})
+
+
 def NewDriver(backend, data):
     auth_token = data["authorizationToken"]["data"]
     data["authorizationToken"].mark_item_as_read_if_equals(
@@ -294,7 +304,8 @@ def ResultConsume(backend, data):
         "serverInfo": {
             "protocolVersion":
                 ".".join(map(str, summary.server.protocol_version)),
-            "agent": summary.server.agent
+            "agent": summary.server.agent,
+            # "address": ":".join(map(str, summary.server.address))
         }
     })
 
@@ -310,3 +321,30 @@ def RetryableNegative(backend, data):
     session_tracker = backend.sessions[key]
     session_tracker.state = '-'
     session_tracker.error_id = data.get('errorId', '')
+
+
+def ForcedRoutingTableUpdate(backend, data):
+    driver_id = data["driverId"]
+    driver = backend.drivers[driver_id]
+    database = data["database"]
+    bookmarks = data["bookmarks"]
+    with driver._pool.refresh_lock:
+        driver._pool.create_routing_table(database)
+        driver._pool.update_routing_table(database=database,
+                                          bookmarks=bookmarks)
+    backend.send_response("Driver", {"id": driver_id})
+
+
+def GetRoutingTable(backend, data):
+    driver_id = data["driverId"]
+    database = data["database"]
+    driver = backend.drivers[driver_id]
+    routing_table = driver._pool.routing_tables[database]
+    response_data = {
+        "database": routing_table.database,
+        "ttl": routing_table.ttl,
+    }
+    for role in ("routers", "readers", "writers"):
+        addresses = routing_table.__getattribute__(role)
+        response_data[role] = list(map(str, addresses))
+    backend.send_response("RoutingTable", response_data)

testkitbackend/skipped_tests.json

@@ -0,0 +1,37 @@
+{
+  "skips": {
+    "stub.routing.Routing.test_should_retry_write_until_success_with_leader_change_using_tx_function":
+      "Driver closes connection to router if DNS resolved name not in routing table",
+    "stub.routing.RoutingV3.test_should_retry_write_until_success_with_leader_change_using_tx_function":
+      "Driver closes connection to router if DNS resolved name not in routing table",
+    "stub.routing.RoutingV4.test_should_retry_write_until_success_with_leader_change_using_tx_function":
+      "Driver closes connection to router if DNS resolved name not in routing table",
+    "stub.routing.Routing.test_should_retry_write_until_success_with_leader_shutdown_during_tx_using_tx_function":
+      "Driver closes connection to router if DNS resolved name not in routing table",
+    "stub.routing.RoutingV3.test_should_retry_write_until_success_with_leader_shutdown_during_tx_using_tx_function":
+      "Driver closes connection to router if DNS resolved name not in routing table",
+    "stub.routing.RoutingV4.test_should_retry_write_until_success_with_leader_shutdown_during_tx_using_tx_function":
+      "Driver closes connection to router if DNS resolved name not in routing table",
+    "stub.routing.Routing.test_should_successfully_acquire_rt_when_router_ip_changes":
+      "Test makes assumptions about how verify_connectivity is implemented",
+    "stub.routing.RoutingV3.test_should_successfully_acquire_rt_when_router_ip_changes":
+      "Test makes assumptions about how verify_connectivity is implemented",
+    "stub.routing.RoutingV4.test_should_successfully_acquire_rt_when_router_ip_changes":
+      "Test makes assumptions about how verify_connectivity is implemented",
+    "stub.retry.TestRetryClustering.test_retry_ForbiddenOnReadOnlyDatabase_ChangingWriter":
+      "Test makes assumptions about how verify_connectivity is implemented",
+    "stub.authorization.AuthorizationTests.test_should_retry_on_auth_expired_on_begin_using_tx_function":
+      "Flaky: test requires the driver to contact servers in a specific order",
+    "stub.authorization.AuthorizationTestsV3.test_should_retry_on_auth_expired_on_begin_using_tx_function":
+      "Flaky: test requires the driver to contact servers in a specific order",
+    "stub.authorization.AuthorizationTestsV4.test_should_retry_on_auth_expired_on_begin_using_tx_function":
+      "Flaky: test requires the driver to contact servers in a specific order"
+  },
+  "features": {
+    "AuthorizationExpiredTreatment": true,
+    "Optimization:ImplicitDefaultArguments": true,
+    "Optimization:MinimalResets": "Driver resets some clean connections when put back into pool",
+    "Optimization:ConnectionReuse": true,
+    "Optimization:PullPipelining": true
+  }
+}

testkitbackend/test_config.json

@@ -36,7 +36,9 @@ class ReadWorkload(object):
     def setup_class(cls):
         cls.server = server = RemoteGraphDatabaseServer()
         server.start()
-        cls.driver = GraphDatabase.driver(server.server_uri, auth=server.auth_token, encrypted=server.encrypted)
+        cls.driver = GraphDatabase.driver(server.server_uri,
+                                          auth=server.auth_token,
+                                          encrypted=server.encrypted)
 
     @classmethod
     def teardown_class(cls):
@@ -62,13 +64,25 @@ def _(tx):
             p = {"record_count": record_count, "x": value}
             for record in tx.run(s, p):
                 assert all(x == value for x in record.values())
+                print(value)
 
         return _
 
+    # @mark.parametrize("record_count,record_width,value", product(
+    #     [1, 1000],  # record count
+    #     [1, 10],    # record width
+    #     [1, u'hello, world'],        # value
+    # ))
     @mark.parametrize("record_count,record_width,value", product(
-        [1, 1000],  # record count
-        [1, 10],    # record width
-        [1, u'hello, world'],        # value
+        [10000],  # record count
+        [768],    # record width
+        [1.234],        # value
     ))
     def test_1x1(self, benchmark, record_count, record_width, value):
-        benchmark(self.work(self.uow(record_count, record_width, value)))
+        # benchmark(self.work(self.uow(record_count, record_width, value)))
+        try:
+            import cProfile as profile
+        except ImportError:
+            import profile
+        work = self.work(self.uow(record_count, record_width, value))
+        profile.runctx("work()", globals=globals(), locals=locals())

tests/performance/test_results.py

@@ -19,26 +19,17 @@
 # limitations under the License.
 
 
-from test.integration.tools import IntegrationTestCase
-
-from os import makedirs, remove
-from os.path import basename, dirname, join as path_join, realpath, isfile, expanduser
-import platform
 from unittest import TestCase, SkipTest
-from shutil import copyfile
-from sys import exit, stderr
 
 try:
     from urllib.request import urlretrieve
 except ImportError:
     from urllib import urlretrieve
 
-from boltkit.controller import WindowsController, UnixController
-
 from neo4j import GraphDatabase
 from neo4j.exceptions import AuthError
 
-from test.env import NEO4J_USER, NEO4J_PASSWORD, NEO4J_SERVER_URI
+from tests.env import NEO4J_USER, NEO4J_PASSWORD, NEO4J_SERVER_URI
 
 
 def is_listening(address):

tests/performance/tools.py

@@ -3,4 +3,5 @@ coverage
 pytest
 pytest-benchmark
 pytest-cov
+pytest-mock
 teamcity-messages