Support Python 3

.github/workflows/test.yml

@@ -0,0 +1,37 @@
+name: test-run
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    if: ( github.event_name == 'push' ||
+        github.event.pull_request.head.repo.full_name != github.repository ) &&
+        ( github.repository == 'tarantool/test-run' )
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.8-dev]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: display python version
+      run: python -c "import sys; print(sys.version)"
+    - name: setup dependencies
+      run: |
+       sudo apt update -y
+       sudo apt-get -y install lua5.1 luarocks
+       sudo luarocks install luacheck
+    - name: setup python dependencies
+      run: |
+       pip install -r requirements.txt
+       pip install -r requirements-test.txt
+    - name: run static analysis
+      run: |
+       make lint

Makefile

@@ -5,7 +5,7 @@ default:
 lint: flake8 luacheck
 
 flake8:
-	python2 -m flake8 *.py lib/*.py
+	python -m flake8 *.py lib/*.py
 
 luacheck:
 	luacheck --config .luacheckrc .

dispatcher.py

@@ -7,13 +7,42 @@
 import yaml
 
 import multiprocessing
-from multiprocessing.queues import SimpleQueue
 
-import listeners
-import lib
+# SimpleQueue is available from multiprocessing.queues on
+# all Python versions known at the moment of writting the code
+# (up to 3.9).
+#
+# It was additionally exposed directly from the multiprocessing
+# module since Python 3.3 ([1]).
+#
+# However the mandatory argument 'ctx'
+# (see multiprocessing.get_context()) was added to the constructor
+# of SimpleQueue from multiprocessing.queues since Python 3.4
+# ([2]).
+#
+# So we should import SimpleQueue from multiprocessing on
+# Python 3.3+ (and must to do so on Python 3.4+) to uniformly
+# instantiate it (without constructor arguments).
+#
+# [1]: https://bugs.python.org/issue11836
+# [2]: https://bugs.python.org/issue18999
+try:
+    # Python 3.3+
+    from multiprocessing import SimpleQueue
+except ImportError:
+    # Python 2
+    from multiprocessing.queues import SimpleQueue
+
+from lib import Options
 from lib.utils import set_fd_cloexec
 from lib.worker import WorkerTaskResult, WorkerDone
 from lib.colorer import color_stdout
+from listeners import ArtifactsWatcher
+from listeners import FailWatcher
+from listeners import HangWatcher
+from listeners import LogOutputWatcher
+from listeners import OutputWatcher
+from listeners import StatisticsWatcher
 
 
 class TcpPortDispatcher:
@@ -122,30 +151,27 @@ def kill_all_workers(self):
                 pass
 
     def init_listeners(self):
-        args = lib.Options().args
+        args = Options().args
         watch_hang = args.no_output_timeout >= 0 and \
             not args.gdb and \
             not args.gdbserver and \
             not args.lldb and \
             not args.valgrind
-        watch_fail = not lib.Options().args.is_force
-
-        log_output_watcher = listeners.LogOutputWatcher()
-        self.statistics = listeners.StatisticsWatcher(
-            log_output_watcher.get_logfile)
-        self.artifacts = listeners.ArtifactsWatcher(
-            log_output_watcher.get_logfile)
-        output_watcher = listeners.OutputWatcher()
+        watch_fail = not Options().args.is_force
+
+        log_output_watcher = LogOutputWatcher()
+        self.statistics = StatisticsWatcher(log_output_watcher.get_logfile)
+        self.artifacts = ArtifactsWatcher(log_output_watcher.get_logfile)
+        output_watcher = OutputWatcher()
         self.listeners = [self.statistics, log_output_watcher, output_watcher, self.artifacts]
         if watch_fail:
-            self.fail_watcher = listeners.FailWatcher(
-                self.terminate_all_workers)
+            self.fail_watcher = FailWatcher(self.terminate_all_workers)
             self.listeners.append(self.fail_watcher)
         if watch_hang:
             warn_timeout = 60.0 if args.long else 10.0
-            hang_watcher = listeners.HangWatcher(
-                output_watcher.not_done_worker_ids, self.kill_all_workers,
-                warn_timeout, float(args.no_output_timeout))
+            hang_watcher = HangWatcher(output_watcher.not_done_worker_ids,
+                                       self.kill_all_workers, warn_timeout,
+                                       float(args.no_output_timeout))
             self.listeners.append(hang_watcher)
 
     def run_max_workers(self):
@@ -315,8 +341,8 @@ def flush_ready(self, inputs):
         # leave only output listeners in self.listeners
         new_listeners = []
         for listener in self.listeners:
-            if isinstance(listener, (listeners.LogOutputWatcher,
-                                     listeners.OutputWatcher)):
+            if isinstance(listener, (LogOutputWatcher,
+                                     OutputWatcher)):
                 listener.report_at_timeout = False
                 new_listeners.append(listener)
         self.listeners = new_listeners

lib/__init__.py

@@ -5,7 +5,7 @@
 from lib.options import Options
 from lib.tarantool_server import TarantoolServer
 from lib.unittest_server import UnittestServer
-from utils import warn_unix_sockets_at_start
+from lib.utils import warn_unix_sockets_at_start
 
 
 __all__ = ['Options']

lib/admin_connection.py

@@ -24,10 +24,12 @@
 import re
 import sys
 
-from tarantool_connection import TarantoolConnection
-from tarantool_connection import TarantoolPool
-from tarantool_connection import TarantoolAsyncConnection
+from lib.tarantool_connection import TarantoolConnection
+from lib.tarantool_connection import TarantoolPool
+from lib.tarantool_connection import TarantoolAsyncConnection
 
+from lib.utils import bytes_to_str
+from lib.utils import str_to_bytes
 
 ADMIN_SEPARATOR = '\n'
 
@@ -36,13 +38,13 @@ def get_handshake(sock, length=128, max_try=100):
     """
     Correct way to get tarantool handshake
     """
-    result = ""
+    result = b""
     i = 0
     while len(result) != length and i < max_try:
-        result = "%s%s" % (result, sock.recv(length-len(result)))
+        result = b"%s%s" % (result, sock.recv(length-len(result)))
         # max_try counter for tarantool/gh-1362
         i += 1
-    return result
+    return bytes_to_str(result)
 
 
 class AdminPool(TarantoolPool):
@@ -54,19 +56,19 @@ def _new_connection(self):
             # tarantool/gh-1163
             # 1. raise only if handshake is not full
             # 2. be silent on crashes or if it's server.stop() operation
-            print 'Handshake error {\n', handshake, '\n}'
+            print('Handshake error {\n', handshake, '\n}')
             raise RuntimeError('Broken tarantool console handshake')
         return s
 
 
 class ExecMixIn(object):
     def cmd(self, socket, cmd, silent):
-        socket.sendall(cmd)
+        socket.sendall(str_to_bytes(cmd))
 
         bufsiz = 4096
         res = ""
         while True:
-            buf = socket.recv(bufsiz)
+            buf = bytes_to_str(socket.recv(bufsiz))
             if not buf:
                 break
             res = res + buf

lib/app_server.py

@@ -21,8 +21,8 @@
 from lib.utils import format_process
 from lib.utils import signame
 from lib.utils import warn_unix_socket
-from test import TestRunGreenlet, TestExecutionError
 from threading import Timer
+from lib.test import TestRunGreenlet, TestExecutionError
 
 
 def timeout_handler(server_process, test_timeout):
@@ -38,8 +38,8 @@ def run_server(execs, cwd, server, logfile, retval):
     timer.start()
     stdout, stderr = server.process.communicate()
     timer.cancel()
-    sys.stdout.write(stdout)
-    with open(logfile, 'a') as f:
+    sys.stdout.write_bytes(stdout)
+    with open(logfile, 'ab') as f:
         f.write(stderr)
     retval['returncode'] = server.process.wait()
     server.process = None
@@ -252,7 +252,7 @@ def is_correct(run):
                     test_suite.ini,
                     params=params,
                     conf_name=conf_name
-                ) for conf_name, params in runs.iteritems()
+                ) for conf_name, params in runs.items()
                     if is_correct(conf_name)])
             else:
                 tests.append(AppTest(test_name,

lib/box_connection.py

@@ -25,7 +25,7 @@
 import ctypes
 import socket
 
-from tarantool_connection import TarantoolConnection
+from lib.tarantool_connection import TarantoolConnection
 
 # monkey patch tarantool and msgpack
 from lib.utils import check_libs
@@ -75,11 +75,11 @@ def execute_no_reconnect(self, command, silent=True):
         if not command:
             return
         if not silent:
-            print command
+            print(command)
         cmd = command.replace(SEPARATOR, ' ') + SEPARATOR
         response = self.py_con.call(cmd)
         if not silent:
-            print response
+            print(response)
         return response
 
     def execute(self, command, silent=True):
@@ -88,8 +88,8 @@ def execute(self, command, silent=True):
     def call(self, command, *args):
         if not command:
             return
-        print 'call ', command, args
+        print('call  {} {}'.format(command, args))
         response = self.py_con.call(command, *args)
         result = str(response)
-        print result
+        print(result)
         return result

lib/connpool.py

@@ -10,7 +10,7 @@
 from contextlib import contextmanager
 from functools import wraps
 
-from test import TestRunGreenlet
+from lib.test import TestRunGreenlet
 
 __all__ = ["ConnectionPool", "retry"]
 
@@ -35,9 +35,9 @@ def __init__(self, size, exc_classes=DEFAULT_EXC_CLASSES, keepalive=None):
         self.keepalive = keepalive
         # Exceptions list must be in tuple form to be caught properly
         self.exc_classes = tuple(exc_classes)
-        for i in xrange(size):
+        for i in range(size):
             self.lock.acquire()
-        for i in xrange(size):
+        for i in range(size):
             greenlet = TestRunGreenlet(self._addOne)
             greenlet.start_later(self.SPAWN_FREQUENCY * i)
         if self.keepalive:
@@ -134,14 +134,14 @@ def deco(*args, **kwargs):
             except exc_classes as e:
                 if logger is not None:
                     logger.log(retry_log_level,
-                               retry_log_message.format(f=f.func_name, e=e))
+                               retry_log_message.format(f=f.__name__, e=e))
                 gevent.sleep(interval)
                 failures += 1
                 if max_failures is not None \
                         and failures > max_failures:
                     if logger is not None:
                         logger.log(max_failure_log_level,
                                    max_failure_log_message.format(
-                                       f=f.func_name, e=e))
+                                       f=f.__name__, e=e))
                     raise
     return deco

lib/inspector.py

@@ -6,8 +6,10 @@
 from gevent.lock import Semaphore
 from gevent.server import StreamServer
 
+from lib.utils import bytes_to_str
 from lib.utils import find_port
 from lib.utils import prefix_each_line
+from lib.utils import str_to_bytes
 from lib.colorer import color_stdout
 from lib.colorer import color_log
 from lib.colorer import qa_notice
@@ -77,7 +79,7 @@ def readline(socket, delimiter='\n', size=4096):
 
         while data:
             try:
-                data = socket.recv(size)
+                data = bytes_to_str(socket.recv(size))
             except IOError:
                 # catch instance halt connection refused errors
                 data = ''
@@ -119,7 +121,7 @@ def handle(self, socket, addr):
             color_log("DEBUG: test-run's response for [{}]\n{}\n".format(
                 line, prefix_each_line(' | ', result)),
                 schema='test-run command')
-            socket.sendall(result)
+            socket.sendall(str_to_bytes(result))
 
         self.sem.release()