Repository: tarantool/tarantool-python (language: python)

Issue:
Run all tests under CI
It was blocked by an old tarantool-python submodule in test-run (https://github.com/tarantool/test-run/issues/126) as @kbelyavs says, but now it does not.
Reconnection strategy - round-robin between instances
Use-case: tarantool is deployed as a geo-distributed cluster, and when one DC goes down, application must switch to reserve DC with minimal service disruption.

Proposal 1: add built-in reconnect strategy which would take several connection URI's as a config, and would quickly to establish connection to them in a circle, returning the first succesful one.
Edge-case: if all URIs are unavailable, return error.

Proposal 2: implement overridable connection strategy class (as in tarantool-java), and provide proposal 1 as example.
Reconnection strategy - round-robin between instances
Use-case: tarantool is deployed as a geo-distributed cluster, and when one DC goes down, application must switch to reserve DC with minimal service disruption.

Proposal 1: add built-in reconnect strategy which would take several connection URI's as a config, and would quickly to establish connection to them in a circle, returning the first succesful one.
Edge-case: if all URIs are unavailable, return error.

Proposal 2: implement overridable connection strategy class (as in tarantool-java), and provide proposal 1 as example.


Discussion / hints:

Fix patch:
diff --git a/.gitmodules b/.gitmodules
index 63e76fe2..e69de29b 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,3 +0,0 @@
-[submodule "test-run"]
-	path = test-run
-	url = https://github.com/tarantool/test-run
diff --git a/Makefile b/Makefile
index 4705feaa..c9dd894c 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,6 @@
 .PHONY: test
 test:
 	python setup.py test
-	cd test && ./test-run.py
 coverage:
 	python -m coverage run -p --source=. setup.py test
 cov-html:
diff --git a/README.rst b/README.rst
index 57c8d33c..3b5af060 100644
--- a/README.rst
+++ b/README.rst
@@ -97,7 +97,7 @@ On Linux:
 On Windows:
 
 * Setup a Linux machine with installed tarantool (called ``remote`` later).
-* (on ``remote``) Copy ``unit/suites/lib/tarantool_python_ci.lua`` to
+* (on ``remote``) Copy ``test/suites/lib/tarantool_python_ci.lua`` to
   ``/etc/tarantool/instances.available``.
 * (on ``remote``) Run ``tarantoolctl start tarantool_python_ci``.
 * Set the following environment variables:
diff --git a/setup.py b/setup.py
index c17634f6..2f4746f0 100755
--- a/setup.py
+++ b/setup.py
@@ -39,7 +39,7 @@
 # Test runner
 # python setup.py test
 try:
-    from unit.setup_command import test
+    from test.setup_command import test
     cmdclass["test"] = test
 except ImportError:
     pass
diff --git a/tarantool/mesh_connection.py b/tarantool/mesh_connection.py
index d1ed851b..7a1d37b2 100644
--- a/tarantool/mesh_connection.py
+++ b/tarantool/mesh_connection.py
@@ -35,6 +35,7 @@
 
 
 def parse_uri(uri):
+    # TODO: Support Unix sockets.
     def parse_error(uri, msg):
         msg = 'URI "%s": %s' % (uri, msg)
         return None, msg
@@ -59,31 +60,53 @@ def parse_error(uri, msg):
 
 
 def validate_address(address):
-    messages = []
-
-    if isinstance(address, dict):
-        if "host" not in address:
-            messages.append("host key must be set")
-        elif not isinstance(address["host"], string_types):
-            messages.append("host value must be string type")
-
-        if "port" not in address:
-            messages.append("port is not set")
-        elif not isinstance(address["port"], int):
-            messages.append("port value must be int type")
-        elif address["port"] == 0:
-            messages.append("port value must not be zero")
-        elif address["port"] > 65535:
-            messages.append("port value must not be above 65535")
-    else:
-        messages.append("address must be a dict")
-
-    if messages:
-        messages_str = ', '.join(messages)
-        msg = 'Address %s: %s' % (str(address), messages_str)
-        return None, msg
-
-    return True, None
+    def format_error(address, err):
+        return None, 'Address %s: %s' % (str(address), err)
+
+    if not isinstance(address, dict):
+        return format_error(address, 'address must be a dict')
+
+    if 'port' not in address or address['port'] is None:
+        return format_error(address, 'port is not set or None')
+
+    if isinstance(address['port'], int):
+        # Looks like an inet address.
+
+        # Validate host.
+        if 'host' not in address or address['host'] is None:
+            return format_error(address,
+                                'host is mandatory for an inet address')
+        if not isinstance(address['host'], string_types):
+            return format_error(address,
+                                'host must be a string for an inet address')
+
+        # Validate port.
+        if not isinstance(address['port'], int):
+            return format_error(address,
+                                'port must be an int for an inet address')
+        if address['port'] < 1 or address['port'] > 65535:
+            return format_error(address, 'port must be in range [1, 65535] '
+                                         'for an inet address')
+
+        # Looks okay.
+        return True, None
+    elif isinstance(address['port'], string_types):
+        # Looks like a unix address.
+
+        # Expect no host.
+        if 'host' in address and address['host'] is not None:
+            return format_error(
+                address, 'host must be unset or None for a unix address')
+
+        # Validate port.
+        if not isinstance(address['port'], string_types):
+            return format_error(address,
+                                'port must be a string for a unix address')
+
+        # Looks okay.
+        return True, None
+
+    return format_error(address, 'port must be an int or a string')
 
 
 class RoundRobinStrategy(object):


Test patch:
diff --git a/test-run b/test-run
deleted file mode 160000
index 72f8b893..00000000
--- a/test-run
+++ /dev/null
@@ -1,1 +0,0 @@
-Subproject commit 72f8b89332b552cf20b2143e6dae7fe5042a9f79
diff --git a/test/.tarantoolctl b/test/.tarantoolctl
deleted file mode 100644
index 5c46f8ac..00000000
--- a/test/.tarantoolctl
+++ /dev/null
@@ -1,15 +0,0 @@
--- Options for test-run tarantoolctl
-
-local workdir = os.getenv('TEST_WORKDIR')
-default_cfg = {
-    pid_file   = workdir,
-    wal_dir    = workdir,
-    memtx_dir  = workdir,
-    vinyl_dir  = workdir,
-    log        = workdir,
-    background = false,
-}
-
-instance_dir = workdir
-
--- vim: set ft=lua :
diff --git a/unit/__init__.py b/test/__init__.py
similarity index 100%
rename from unit/__init__.py
rename to test/__init__.py
diff --git a/test/cluster-py/instance.lua b/test/cluster-py/instance.lua
deleted file mode 100644
index 2fcba42e..00000000
--- a/test/cluster-py/instance.lua
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/usr/bin/env tarantool
-
-local INSTANCE_ID = string.match(arg[0], "%d")
-local SOCKET_DIR = require('fio').cwd()
-
-local function instance_uri(instance_id)
-    return SOCKET_DIR..'/instance'..instance_id..'.sock';
-end
-
-require('console').listen(os.getenv('ADMIN'))
-
-box.cfg({
-    --listen              = os.getenv("LISTEN"),
-    listen = instance_uri(INSTANCE_ID),
-    memtx_memory        = 107374182,
-})
diff --git a/test/cluster-py/instance1.lua b/test/cluster-py/instance1.lua
deleted file mode 120000
index 2087830f..00000000
--- a/test/cluster-py/instance1.lua
+++ /dev/null
@@ -1,1 +0,0 @@
-instance.lua
\ No newline at end of file
diff --git a/test/cluster-py/instance2.lua b/test/cluster-py/instance2.lua
deleted file mode 120000
index 2087830f..00000000
--- a/test/cluster-py/instance2.lua
+++ /dev/null
@@ -1,1 +0,0 @@
-instance.lua
\ No newline at end of file
diff --git a/test/cluster-py/master.lua b/test/cluster-py/master.lua
deleted file mode 100644
index b43bafd5..00000000
--- a/test/cluster-py/master.lua
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/usr/bin/env tarantool
-local os = require('os')
-box.cfg({
-    listen              = os.getenv("LISTEN"),
-    memtx_memory        = 107374182,
-    replication_timeout = 0.1
-})
-
-require('console').listen(os.getenv('ADMIN'))
diff --git a/test/cluster-py/multi.result b/test/cluster-py/multi.result
deleted file mode 100644
index 5e3a8573..00000000
--- a/test/cluster-py/multi.result
+++ /dev/null
@@ -1,22 +0,0 @@
-box.schema.user.grant('guest', 'read,write,execute', 'universe')
----
-...
-_ = box.schema.space.create('test')
----
-...
-_ = box.space.test:create_index('primary')
----
-...
-box.schema.user.grant('guest', 'read,write,execute', 'universe')
----
-...
-_ = box.schema.space.create('test')
----
-...
-_ = box.space.test:create_index('primary')
----
-...
-- [1, 0]
-- [1, 1]
-- [1, 0]
-NetworkError !
diff --git a/test/cluster-py/multi.test.py b/test/cluster-py/multi.test.py
deleted file mode 100644
index 0db520b0..00000000
--- a/test/cluster-py/multi.test.py
+++ /dev/null
@@ -1,71 +0,0 @@
-import sys
-import os
-import time
-import yaml
-from lib.tarantool_server import TarantoolServer
-sys.path.append('../tarantool')
-from mesh_connection import MeshConnection
-from tarantool.const import (
-    SOCKET_TIMEOUT,
-    RECONNECT_DELAY,
-)
-from tarantool.error import NetworkError
-from tarantool.utils import ENCODING_DEFAULT
-
-INSTANCE_N = 2
-
-
-def check_connection(con):
-    try:
-        s = con.space('test')
-        print(s.select())
-    except NetworkError:
-        print('NetworkError !')
-    except Exception as e:
-        print(e)
-
-
-# Start instances
-master = server
-cluster = [master]
-for i in range(INSTANCE_N):
-    server = TarantoolServer(server.ini)
-    server.script = 'cluster-py/instance%d.lua' % (i+1)
-    server.vardir = os.path.join(server.vardir, 'instance', str(i))
-    server.deploy()
-    server.admin("box.schema.user.grant('guest', 'read,write,execute', 'universe')")
-    server.admin("_ = box.schema.space.create('test')")
-    server.admin("_ = box.space.test:create_index('primary')")
-    server.admin("box.space.test:insert{%d, %s}" % (1, i), silent = True)
-    cluster.append(server)
-
-# Make a list of servers
-sources = []
-for server in cluster[1:]:
-    sources.append(yaml.safe_load(server.admin('box.cfg.listen', silent=True))[0])
-
-addrs = []
-for addr in sources:
-    addrs.append({'host': None, 'port': addr})
-
-con = MeshConnection(addrs=addrs,
-                     user=None,
-                     password=None,
-                     socket_timeout=SOCKET_TIMEOUT,
-                     reconnect_max_attempts=0,
-                     reconnect_delay=RECONNECT_DELAY,
-                     connect_now=True,
-                     encoding=ENCODING_DEFAULT)
-
-cluster[0].stop()       # stop server - no effect
-check_connection(con)   # instance#1
-cluster[1].stop()       # stop instance#1
-check_connection(con)   # instance#2
-cluster[1].start()      # start instance#1
-cluster[2].stop()       # stop instance#2
-check_connection(con)   # instance#1 again
-cluster[1].stop()       # stop instance#1
-check_connection(con)   # both stopped: NetworkError !
-
-master.cleanup()
-master.deploy()
diff --git a/test/cluster-py/suite.ini b/test/cluster-py/suite.ini
deleted file mode 100644
index a2fea18b..00000000
--- a/test/cluster-py/suite.ini
+++ /dev/null
@@ -1,5 +0,0 @@
-[default]
-core = tarantool
-script =  master.lua
-description = reconnect
-is_parallel = False
diff --git a/unit/setup_command.py b/test/setup_command.py
similarity index 86%
rename from unit/setup_command.py
rename to test/setup_command.py
index 65e6f780..1711e2d1 100755
--- a/unit/setup_command.py
+++ b/test/setup_command.py
@@ -10,7 +10,7 @@
 
 class test(setuptools.Command):
     user_options = []
-    description = 'Run unit tests'
+    description = 'Run tests'
 
     def initialize_options(self):
         pass
@@ -23,7 +23,7 @@ def run(self):
         Find all tests in test/tarantool/ and run them
         '''
 
-        tests = unittest.defaultTestLoader.discover('unit', pattern='suites')
+        tests = unittest.defaultTestLoader.discover('test', pattern='suites')
         test_runner = unittest.TextTestRunner(verbosity=2)
         result = test_runner.run(tests)
         if not result.wasSuccessful():
diff --git a/unit/suites/__init__.py b/test/suites/__init__.py
similarity index 100%
rename from unit/suites/__init__.py
rename to test/suites/__init__.py
diff --git a/unit/suites/box.lua b/test/suites/box.lua
similarity index 100%
rename from unit/suites/box.lua
rename to test/suites/box.lua
diff --git a/unit/suites/lib/__init__.py b/test/suites/lib/__init__.py
similarity index 100%
rename from unit/suites/lib/__init__.py
rename to test/suites/lib/__init__.py
diff --git a/unit/suites/lib/remote_tarantool_server.py b/test/suites/lib/remote_tarantool_server.py
similarity index 100%
rename from unit/suites/lib/remote_tarantool_server.py
rename to test/suites/lib/remote_tarantool_server.py
diff --git a/unit/suites/lib/tarantool_admin.py b/test/suites/lib/tarantool_admin.py
similarity index 100%
rename from unit/suites/lib/tarantool_admin.py
rename to test/suites/lib/tarantool_admin.py
diff --git a/unit/suites/lib/tarantool_python_ci.lua b/test/suites/lib/tarantool_python_ci.lua
similarity index 100%
rename from unit/suites/lib/tarantool_python_ci.lua
rename to test/suites/lib/tarantool_python_ci.lua
diff --git a/unit/suites/lib/tarantool_server.py b/test/suites/lib/tarantool_server.py
similarity index 99%
rename from unit/suites/lib/tarantool_server.py
rename to test/suites/lib/tarantool_server.py
index 4fc1de28..440a5d3c 100644
--- a/unit/suites/lib/tarantool_server.py
+++ b/test/suites/lib/tarantool_server.py
@@ -194,6 +194,7 @@ def start(self):
         self.wait_until_started()
 
     def stop(self):
+        self.admin.disconnect()
         if self.process.poll() is None:
             self.process.terminate()
             self.process.wait()
diff --git a/unit/suites/test_dbapi.py b/test/suites/test_dbapi.py
similarity index 98%
rename from unit/suites/test_dbapi.py
rename to test/suites/test_dbapi.py
index 39672b5d..074620f1 100644
--- a/unit/suites/test_dbapi.py
+++ b/test/suites/test_dbapi.py
@@ -26,7 +26,7 @@ def setUpClass(self):
         print(' DBAPI '.center(70, '='), file=sys.stderr)
         print('-' * 70, file=sys.stderr)
         self.srv = TarantoolServer()
-        self.srv.script = 'unit/suites/box.lua'
+        self.srv.script = 'test/suites/box.lua'
         self.srv.start()
         self.con = tarantool.Connection(self.srv.host, self.srv.args['primary'])
         self.driver = dbapi
diff --git a/unit/suites/test_dml.py b/test/suites/test_dml.py
similarity index 99%
rename from unit/suites/test_dml.py
rename to test/suites/test_dml.py
index 31e821ea..e3922a10 100644
--- a/unit/suites/test_dml.py
+++ b/test/suites/test_dml.py
@@ -14,7 +14,7 @@ def setUpClass(self):
         print(' DML '.center(70, '='), file=sys.stderr)
         print('-' * 70, file=sys.stderr)
         self.srv = TarantoolServer()
-        self.srv.script = 'unit/suites/box.lua'
+        self.srv.script = 'test/suites/box.lua'
         self.srv.start()
         self.con = tarantool.Connection(self.srv.host, self.srv.args['primary'])
         self.adm = self.srv.admin
diff --git a/unit/suites/test_execute.py b/test/suites/test_execute.py
similarity index 98%
rename from unit/suites/test_execute.py
rename to test/suites/test_execute.py
index 21b8cfac..e10c0073 100644
--- a/unit/suites/test_execute.py
+++ b/test/suites/test_execute.py
@@ -26,7 +26,7 @@ def setUpClass(self):
         print(' EXECUTE '.center(70, '='), file=sys.stderr)
         print('-' * 70, file=sys.stderr)
         self.srv = TarantoolServer()
-        self.srv.script = 'unit/suites/box.lua'
+        self.srv.script = 'test/suites/box.lua'
         self.srv.start()
         self.con = tarantool.Connection(self.srv.host, self.srv.args['primary'])
 
diff --git a/unit/suites/test_mesh.py b/test/suites/test_mesh.py
similarity index 87%
rename from unit/suites/test_mesh.py
rename to test/suites/test_mesh.py
index dda59a89..df0b059a 100644
--- a/unit/suites/test_mesh.py
+++ b/test/suites/test_mesh.py
@@ -8,7 +8,9 @@
 from time import sleep
 import tarantool
 from tarantool.error import (
+    NetworkError,
     ConfigurationError,
+    NetworkWarning,
     ClusterDiscoveryWarning,
 )
 from .lib.tarantool_server import TarantoolServer
@@ -16,7 +18,7 @@
 
 def create_server(_id):
     srv = TarantoolServer()
-    srv.script = 'unit/suites/box.lua'
+    srv.script = 'test/suites/box.lua'
     srv.start()
     srv.admin("box.schema.user.create('test', {password = 'test', " +
               "if_not_exists = true})")
@@ -70,6 +72,40 @@ def setUp(self):
         self.define_cluster_function(self.get_all_nodes_func_name,
                                      self.servers)
 
+    def test_00_basic(self):
+        def assert_srv_id(con, srv_id):
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', category=NetworkWarning)
+                resp = con.call('srv_id')
+            self.assertEqual(resp.data and resp.data[0], srv_id)
+
+        con = tarantool.MeshConnection(addrs=[
+            {'host': self.host_1, 'port': self.port_1},
+            {'host': self.host_2, 'port': self.port_2},
+        ], user='test', password='test')
+
+        # Response from instance#1.
+        assert_srv_id(con, 1)
+
+        # Stop instance#1 -- response from instance#2.
+        self.srv.stop()
+        assert_srv_id(con, 2)
+
+        # Start instance#1, stop instance#2 -- response from
+        # instance#1 again.
+        self.srv.start()
+        self.srv.admin('function srv_id() return 1 end')
+        self.srv2.stop()
+        assert_srv_id(con, 1)
+
+        # Stop instance #2 -- NetworkError (because we have no
+        # alive servers anymore).
+        self.srv.stop()
+        with self.assertRaises(NetworkError):
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', category=NetworkWarning)
+                con.ping()
+
     def test_01_contructor(self):
         # Verify that an error is risen when no addresses are
         # configured (neither with host/port, nor with addrs).
diff --git a/unit/suites/test_protocol.py b/test/suites/test_protocol.py
similarity index 100%
rename from unit/suites/test_protocol.py
rename to test/suites/test_protocol.py
diff --git a/unit/suites/test_reconnect.py b/test/suites/test_reconnect.py
similarity index 98%
rename from unit/suites/test_reconnect.py
rename to test/suites/test_reconnect.py
index 24470bae..403fd351 100644
--- a/unit/suites/test_reconnect.py
+++ b/test/suites/test_reconnect.py
@@ -15,7 +15,7 @@ def setUpClass(self):
         print(' RECONNECT '.center(70, '='), file=sys.stderr)
         print('-' * 70, file=sys.stderr)
         self.srv = TarantoolServer()
-        self.srv.script = 'unit/suites/box.lua'
+        self.srv.script = 'test/suites/box.lua'
 
     def setUp(self):
         # prevent a remote tarantool from clean our session
diff --git a/unit/suites/test_schema.py b/test/suites/test_schema.py
similarity index 99%
rename from unit/suites/test_schema.py
rename to test/suites/test_schema.py
index 37850e06..c5f12282 100644
--- a/unit/suites/test_schema.py
+++ b/test/suites/test_schema.py
@@ -43,7 +43,7 @@ def setUpClass(self):
         print(' SCHEMA ({}) '.format(params).center(70, '='), file=sys.stderr)
         print('-' * 70, file=sys.stderr)
         self.srv = TarantoolServer()
-        self.srv.script = 'unit/suites/box.lua'
+        self.srv.script = 'test/suites/box.lua'
         self.srv.start()
         self.con = tarantool.Connection(self.srv.host, self.srv.args['primary'],
                                         encoding=self.encoding)
diff --git a/test/test-run.py b/test/test-run.py
deleted file mode 120000
index 21bb780a..00000000
--- a/test/test-run.py
+++ /dev/null
@@ -1,1 +0,0 @@
-../test-run/test-run.py
\ No newline at end of file