Repository: sql-machine-learning/elasticdl

Issue:
Master makes sure Redis service starts successfully
Now it may fail:
 [embedding_service.py:206:start_embedding_pod_and_redis] Starting pod for embedding service ...
Could not connect to Redis at 11.238.113.73:30001: Connection refused
[2019-08-26 08:02:09,209] [INFO] [main.py:190:main] Embedding service start succeeded. The endpoint is {'11.238.113.73': [30001, 30002, 30003, 30004, 30005, 30006]}.
[2019-08-26 08:02:09,209] [INFO] [main.py:194:main] Starting master service


Discussion:

Resolving patch:
diff --git a/elasticdl/python/common/constants.py b/elasticdl/python/common/constants.py
--- a/elasticdl/python/common/constants.py
+++ b/elasticdl/python/common/constants.py
@@ -29,3 +29,7 @@ class Mode(object):
     TRAINING = "training"
     EVALUATION = "evaluation"
     PREDICTION = "prediction"
+
+
+class Redis(object):
+    MAX_COMMAND_RETRY_TIMES = 10
diff --git a/elasticdl/python/common/embedding_service.py b/elasticdl/python/common/embedding_service.py
--- a/elasticdl/python/common/embedding_service.py
+++ b/elasticdl/python/common/embedding_service.py
@@ -1,4 +1,5 @@
 import argparse
+import os
 import subprocess
 import time
 
@@ -7,6 +8,7 @@
 
 from elasticdl.python.common import k8s_client as k8s
 from elasticdl.python.common.args import pos_int
+from elasticdl.python.common.constants import Redis
 from elasticdl.python.common.log_util import default_logger as logger
 
 
@@ -85,37 +87,41 @@ def _create_redis_cluster(self):
                 for port in port_list
             ]
         )
-        try:
-            command = (
-                "echo yes | redis-cli --cluster create %s "
-                "--cluster-replicas %d"
-                % (redis_cluster_command, self._replicas)
-            )
-            redis_process = subprocess.Popen(
-                [command], shell=True, stdout=subprocess.DEVNULL
-            )
-            redis_process.wait()
-        except Exception as e:
-            logger.error(e)
-            return None
-        else:
+        command = (
+            "echo yes | redis-cli --cluster create %s "
+            "--cluster-replicas %d" % (redis_cluster_command, self._replicas)
+        )
+        returncode = self._run_shell_command(command)
+        if not returncode:
             return self._embedding_service_endpoint
+        raise Exception(
+            "Create Redis cluster failed with command: %s" % command
+        )
 
     def stop_embedding_service(self, save="nosave"):
+        failed_redis_nodes = []
         for redis_node in [
             "-h %s -p %d" % (ip, port)
             for ip, port_list in self._embedding_service_endpoint.items()
             for port in port_list
         ]:
-            try:
-                command = "redis-cli %s shutdown %s" % (redis_node, save)
-                redis_process = subprocess.Popen(
-                    [command], shell=True, stdout=subprocess.DEVNULL
+            command = "redis-cli %s shutdown %s" % (redis_node, save)
+            returncode = self._run_shell_command(command)
+            if returncode:
+                failed_redis_nodes.append(redis_node)
+
+        if failed_redis_nodes:
+            failed_redis_nodes = [i.split(" ") for i in failed_redis_nodes]
+            logger.info(
+                "Stop these redis nodes failed: %s."
+                % ";".join(
+                    [
+                        "%s:%s" % (redis_node[1], redis_node[3])
+                        for redis_node in failed_redis_nodes
+                    ]
                 )
-                redis_process.wait()
-            except Exception as e:
-                logger.error(e)
-                return False
+            )
+            return False
 
         return True
 
@@ -172,6 +178,7 @@ def start_redis_service(self):
                 args.cluster_node_timeout,
             )
         )
+        failed_port = []
         for i in range(args.num_of_redis_instances):
             port = args.first_port + i
             command = (
@@ -182,10 +189,38 @@ def start_redis_service(self):
                 "--protected-mode no"
                 % (port, port, args.cluster_node_timeout, port, port, port)
             )
+            returncode = self._run_shell_command(command)
+            if returncode:
+                failed_port.append(port)
+        if failed_port:
+            local_ip = os.getenv("MY_POD_IP", "localhost")
+            logger.info(
+                "%s starts these redis instances failed: %s"
+                % (local_ip, ";".join(map(str, failed_port)))
+            )
+
+    def _run_shell_command(self, command):
+        retry_times = 0
+        while retry_times <= Redis.MAX_COMMAND_RETRY_TIMES:
+            if retry_times:
+                logger.warning(
+                    'Command: "%s" failed to run, retry times: %d .'
+                    % (command, retry_times)
+                )
             redis_process = subprocess.Popen(
-                [command], shell=True, stdout=subprocess.DEVNULL
+                [command],
+                shell=True,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
             )
             redis_process.wait()
+            if not redis_process.returncode:
+                break
+            redis_process.kill()
+            # Wait for retry
+            time.sleep(1)
+            retry_times += 1
+        return redis_process.returncode
 
     # TODO: Now, we use single pod to start redis cluster service, we
     # should support a redis cluster service running on multi-pods in
@@ -256,7 +291,6 @@ def lookup_embedding(
             for ip, port_list in embedding_service_endpoint.items()
             for port in port_list
         ]
-        embedding_vectors = []
         embedding_service = RedisCluster(
             startup_nodes=startup_nodes, decode_responses=False
         ).pipeline()


Test patch: