Repository: ray-project/ray

Issue:
Test failure in Jenkins "Exception: Expected 10 Redis shard addresses, found 7".
The following error occurred in Jenkins in this log https://amplab.cs.berkeley.edu/jenkins/job/Ray-PRB/1453/console.

```
+ python /home/jenkins/workspace/Ray-PRB/test/jenkins_tests/multi_node_docker_test.py --docker-image=60125e3f54e6f8f0623b956c8cb04ddc6402819ed22ef5908c5b9ba8db38b46d --num-nodes=5 --num-redis-shards=10 --test-script=/ray/test/jenkins_tests/multi_node_tests/test_0.py
Starting head node with command:['docker', 'run', '-d', '--shm-size=1G', '60125e3f54e6f8f0623b956c8cb04ddc6402819ed22ef5908c5b9ba8db38b46d', 'ray', 'start', '--head', '--block', '--redis-port=6379', '--num-redis-shards=10', '--num-cpus=10', '--num-gpus=0']
Starting worker node with command:['docker', 'run', '-d', '--shm-size=1G', '--shm-size=1G', '60125e3f54e6f8f0623b956c8cb04ddc6402819ed22ef5908c5b9ba8db38b46d', 'ray', 'start', '--block', '--redis-address=172.17.0.12:6379', '--num-cpus=10', '--num-gpus=0']
Starting worker node with command:['docker', 'run', '-d', '--shm-size=1G', '--shm-size=1G', '60125e3f54e6f8f0623b956c8cb04ddc6402819ed22ef5908c5b9ba8db38b46d', 'ray', 'start', '--block', '--redis-address=172.17.0.12:6379', '--num-cpus=10', '--num-gpus=0']
Starting worker node with command:['docker', 'run', '-d', '--shm-size=1G', '--shm-size=1G', '60125e3f54e6f8f0623b956c8cb04ddc6402819ed22ef5908c5b9ba8db38b46d', 'ray', 'start', '--block', '--redis-address=172.17.0.12:6379', '--num-cpus=10', '--num-gpus=0']
Starting worker node with command:['docker', 'run', '-d', '--shm-size=1G', '--shm-size=1G', '60125e3f54e6f8f0623b956c8cb04ddc6402819ed22ef5908c5b9ba8db38b46d', 'ray', 'start', '--block', '--redis-address=172.17.0.12:6379', '--num-cpus=10', '--num-gpus=0']
Starting driver with command /ray/test/jenkins_tests/multi_node_tests/test_0.py.
STDOUT:
Driver 0 started at 1501449715.55.

STDERR:
Traceback (most recent call last):
  File "/ray/test/jenkins_tests/multi_node_tests/test_0.py", line 22, in <module>
    ray.init(redis_address=redis_address)
  File "/opt/conda/lib/python2.7/site-packages/ray-0.1.2-py2.7-linux-x86_64.egg/ray/worker.py", line 1011, in init
    num_gpus=num_gpus, num_redis_shards=num_redis_shards)
  File "/opt/conda/lib/python2.7/site-packages/ray-0.1.2-py2.7-linux-x86_64.egg/ray/worker.py", line 962, in _init
    mode=driver_mode, worker=global_worker, actor_id=NIL_ACTOR_ID)
  File "/opt/conda/lib/python2.7/site-packages/ray-0.1.2-py2.7-linux-x86_64.egg/ray/worker.py", line 1338, in connect
    global_state._initialize_global_state(redis_ip_address, int(redis_port))
  File "/opt/conda/lib/python2.7/site-packages/ray-0.1.2-py2.7-linux-x86_64.egg/ray/experimental/state.py", line 96, in _initialize_global_state
    len(ip_address_ports)))
Exception: Expected 10 Redis shard addresses, found 7

stop_node {'container_id': u'fea4409391126e14fca2459ce705051d644ec15163d5143708fc7820a2b7ba2a', 'is_head': True}
stop_node {'container_id': u'd9942971920d496cd39b65b4b9f95525a4479d455036ec2460fdbe8fe4b51c6e', 'is_head': False}
Traceback (most recent call last):
  File "/home/jenkins/workspace/Ray-PRB/test/jenkins_tests/multi_node_docker_test.py", line 323, in <module>
    d.stop_ray()
  File "/home/jenkins/workspace/Ray-PRB/test/jenkins_tests/multi_node_docker_test.py", line 224, in stop_ray
    self._stop_node(container_id)
  File "/home/jenkins/workspace/Ray-PRB/test/jenkins_tests/multi_node_docker_test.py", line 207, in _stop_node
    .format(container_id))
Exception: Failed to stop container 27327225b793cc721628b2fa4cf66b3c0a49eb5f0732273b66fa7ee505689655.
Build step 'Execute shell' marked build as failure
Test FAILed.
Refer to this link for build results (access rights to CI server needed): 
https://amplab.cs.berkeley.edu/jenkins//job/Ray-PRB/1453/
Test FAILed.
Finished: FAILURE
```


Discussion:

Resolving patch:
diff --git a/python/ray/experimental/state.py b/python/ray/experimental/state.py
--- a/python/ray/experimental/state.py
+++ b/python/ray/experimental/state.py
@@ -57,6 +57,7 @@ class GlobalState(object):
     def __init__(self):
         """Create a GlobalState object."""
         self.redis_client = None
+        self.redis_clients = None
 
     def _check_connected(self):
         """Check that the object has been initialized before it is used.
@@ -69,32 +70,65 @@ def _check_connected(self):
             raise Exception("The ray.global_state API cannot be used before "
                             "ray.init has been called.")
 
-    def _initialize_global_state(self, redis_ip_address, redis_port):
+        if self.redis_clients is None:
+            raise Exception("The ray.global_state API cannot be used before "
+                            "ray.init has been called.")
+
+    def _initialize_global_state(self, redis_ip_address, redis_port,
+                                 timeout=20):
         """Initialize the GlobalState object by connecting to Redis.
 
+        It's possible that certain keys in Redis may not have been fully
+        populated yet. In this case, we will retry this method until they have
+        been populated or we exceed a timeout.
+
         Args:
             redis_ip_address: The IP address of the node that the Redis server
                 lives on.
             redis_port: The port that the Redis server is listening on.
+            timeout: The maximum amount of time (in seconds) that we should
+                wait for the keys in Redis to be populated.
         """
         self.redis_client = redis.StrictRedis(host=redis_ip_address,
                                               port=redis_port)
-        self.redis_clients = []
-        num_redis_shards = self.redis_client.get("NumRedisShards")
-        if num_redis_shards is None:
-            raise Exception("No entry found for NumRedisShards")
-        num_redis_shards = int(num_redis_shards)
-        if (num_redis_shards < 1):
-            raise Exception("Expected at least one Redis shard, found "
-                            "{}.".format(num_redis_shards))
-
-        ip_address_ports = self.redis_client.lrange("RedisShards", start=0,
-                                                    end=-1)
-        if len(ip_address_ports) != num_redis_shards:
-            raise Exception("Expected {} Redis shard addresses, found "
-                            "{}".format(num_redis_shards,
-                                        len(ip_address_ports)))
 
+        start_time = time.time()
+
+        num_redis_shards = None
+        ip_address_ports = []
+
+        while time.time() - start_time < timeout:
+            # Attempt to get the number of Redis shards.
+            num_redis_shards = self.redis_client.get("NumRedisShards")
+            if num_redis_shards is None:
+                print("Waiting longer for NumRedisShards to be populated.")
+                time.sleep(1)
+                continue
+            num_redis_shards = int(num_redis_shards)
+            if (num_redis_shards < 1):
+                raise Exception("Expected at least one Redis shard, found "
+                                "{}.".format(num_redis_shards))
+
+            # Attempt to get all of the Redis shards.
+            ip_address_ports = self.redis_client.lrange("RedisShards", start=0,
+                                                        end=-1)
+            if len(ip_address_ports) != num_redis_shards:
+                print("Waiting longer for RedisShards to be populated.")
+                time.sleep(1)
+                continue
+
+            # If we got here then we successfully got all of the information.
+            break
+
+        # Check to see if we timed out.
+        if time.time() - start_time >= timeout:
+            raise Exception("Timed out while attempting to initialize the "
+                            "global state. num_redis_shards = {}, "
+                            "ip_address_ports = {}"
+                            .format(num_redis_shards, ip_address_ports))
+
+        # Get the rest of the information.
+        self.redis_clients = []
         for ip_address_port in ip_address_ports:
             shard_address, shard_port = ip_address_port.split(b":")
             self.redis_clients.append(redis.StrictRedis(host=shard_address,


Test patch: