# Buggy code
def _initialize_global_state(self, redis_ip_address, redis_port):
    """Initialize the GlobalState object by connecting to Redis.

    Args:
        redis_ip_address: The IP address of the node that the Redis server
            lives on.
        redis_port: The port that the Redis server is listening on.
    """
    self.redis_client = redis.StrictRedis(host=redis_ip_address, port=redis_port)
    self.redis_clients = []
    num_redis_shards = self.redis_client.get("NumRedisShards")
    if num_redis_shards is None:
        raise Exception("No entry found for NumRedisShards")
    num_redis_shards = int(num_redis_shards)
    if num_redis_shards < 1:
        raise Exception(
            "Expected at least one Redis shard, found {}.".format(num_redis_shards)
        )

    ip_address_ports = self.redis_client.lrange("RedisShards", start=0, end=-1)
    if len(ip_address_ports) != num_redis_shards:
        raise Exception(
            "Expected {} Redis shard addresses, found {}".format(
                num_redis_shards, len(ip_address_ports)
            )
        )

    for ip_address_port in ip_address_ports:
        shard_address, shard_port = ip_address_port.split(b":")
        self.redis_clients.append(
            redis.StrictRedis(host=shard_address, port=shard_port)
        )


# Traceback (Exception)
+ python /home/jenkins/workspace/Ray-PRB/test/jenkins_tests/multi_node_docker_test.py --docker-image=60125e3f54e6f8f0623b956c8cb04ddc6402819ed22ef5908c5b9ba8db38b46d --num-nodes=5 --num-redis-shards=10 --test-script=/ray/test/jenkins_tests/multi_node_tests/test_0.py
Starting head node with command:['docker', 'run', '-d', '--shm-size=1G', '60125e3f54e6f8f0623b956c8cb04ddc6402819ed22ef5908c5b9ba8db38b46d', 'ray', 'start', '--head', '--block', '--redis-port=6379', '--num-redis-shards=10', '--num-cpus=10', '--num-gpus=0']
Starting worker node with command:['docker', 'run', '-d', '--shm-size=1G', '--shm-size=1G', '60125e3f54e6f8f0623b956c8cb04ddc6402819ed22ef5908c5b9ba8db38b46d', 'ray', 'start', '--block', '--redis-address=172.17.0.12:6379', '--num-cpus=10', '--num-gpus=0']
Starting worker node with command:['docker', 'run', '-d', '--shm-size=1G', '--shm-size=1G', '60125e3f54e6f8f0623b956c8cb04ddc6402819ed22ef5908c5b9ba8db38b46d', 'ray', 'start', '--block', '--redis-address=172.17.0.12:6379', '--num-cpus=10', '--num-gpus=0']
Starting worker node with command:['docker', 'run', '-d', '--shm-size=1G', '--shm-size=1G', '60125e3f54e6f8f0623b956c8cb04ddc6402819ed22ef5908c5b9ba8db38b46d', 'ray', 'start', '--block', '--redis-address=172.17.0.12:6379', '--num-cpus=10', '--num-gpus=0']
Starting worker node with command:['docker', 'run', '-d', '--shm-size=1G', '--shm-size=1G', '60125e3f54e6f8f0623b956c8cb04ddc6402819ed22ef5908c5b9ba8db38b46d', 'ray', 'start', '--block', '--redis-address=172.17.0.12:6379', '--num-cpus=10', '--num-gpus=0']
Starting driver with command /ray/test/jenkins_tests/multi_node_tests/test_0.py.
STDOUT:
Driver 0 started at 1501449715.55.

STDERR:
Traceback (most recent call last):
File "/ray/test/jenkins_tests/multi_node_tests/test_0.py", line 22, in <module>
ray.init(redis_address=redis_address)
File "/opt/conda/lib/python2.7/site-packages/ray-0.1.2-py2.7-linux-x86_64.egg/ray/worker.py", line 1011, in init
num_gpus=num_gpus, num_redis_shards=num_redis_shards)
File "/opt/conda/lib/python2.7/site-packages/ray-0.1.2-py2.7-linux-x86_64.egg/ray/worker.py", line 962, in _init
mode=driver_mode, worker=global_worker, actor_id=NIL_ACTOR_ID)
File "/opt/conda/lib/python2.7/site-packages/ray-0.1.2-py2.7-linux-x86_64.egg/ray/worker.py", line 1338, in connect
global_state._initialize_global_state(redis_ip_address, int(redis_port))
File "/opt/conda/lib/python2.7/site-packages/ray-0.1.2-py2.7-linux-x86_64.egg/ray/experimental/state.py", line 96, in _initialize_global_state
len(ip_address_ports)))
Exception: Expected 10 Redis shard addresses, found 7

stop_node {'container_id': u'fea4409391126e14fca2459ce705051d644ec15163d5143708fc7820a2b7ba2a', 'is_head': True}
stop_node {'container_id': u'd9942971920d496cd39b65b4b9f95525a4479d455036ec2460fdbe8fe4b51c6e', 'is_head': False}
Traceback (most recent call last):
File "/home/jenkins/workspace/Ray-PRB/test/jenkins_tests/multi_node_docker_test.py", line 323, in <module>
d.stop_ray()
File "/home/jenkins/workspace/Ray-PRB/test/jenkins_tests/multi_node_docker_test.py", line 224, in stop_ray
self._stop_node(container_id)
File "/home/jenkins/workspace/Ray-PRB/test/jenkins_tests/multi_node_docker_test.py", line 207, in _stop_node
.format(container_id))
Exception: Failed to stop container 27327225b793cc721628b2fa4cf66b3c0a49eb5f0732273b66fa7ee505689655.
Build step 'Execute shell' marked build as failure
Test FAILed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/Ray-PRB/1453/
Test FAILed.
Finished: FAILURE

# Fixed code
def _initialize_global_state(self, redis_ip_address, redis_port, timeout=20):
    """Initialize the GlobalState object by connecting to Redis.

    It's possible that certain keys in Redis may not have been fully
    populated yet. In this case, we will retry this method until they have
    been populated or we exceed a timeout.

    Args:
        redis_ip_address: The IP address of the node that the Redis server
            lives on.
        redis_port: The port that the Redis server is listening on.
        timeout: The maximum amount of time (in seconds) that we should
            wait for the keys in Redis to be populated.
    """
    self.redis_client = redis.StrictRedis(host=redis_ip_address, port=redis_port)

    start_time = time.time()

    num_redis_shards = None
    ip_address_ports = []

    while time.time() - start_time < timeout:
        # Attempt to get the number of Redis shards.
        num_redis_shards = self.redis_client.get("NumRedisShards")
        if num_redis_shards is None:
            print("Waiting longer for NumRedisShards to be populated.")
            time.sleep(1)
            continue
        num_redis_shards = int(num_redis_shards)
        if num_redis_shards < 1:
            raise Exception(
                "Expected at least one Redis shard, found {}.".format(num_redis_shards)
            )

        # Attempt to get all of the Redis shards.
        ip_address_ports = self.redis_client.lrange("RedisShards", start=0, end=-1)
        if len(ip_address_ports) != num_redis_shards:
            print("Waiting longer for RedisShards to be populated.")
            time.sleep(1)
            continue

        # If we got here then we successfully got all of the information.
        break

    # Check to see if we timed out.
    if time.time() - start_time >= timeout:
        raise Exception(
            "Timed out while attempting to initialize the "
            "global state. num_redis_shards = {}, "
            "ip_address_ports = {}".format(num_redis_shards, ip_address_ports)
        )

    # Get the rest of the information.
    self.redis_clients = []
    for ip_address_port in ip_address_ports:
        shard_address, shard_port = ip_address_port.split(b":")
        self.redis_clients.append(
            redis.StrictRedis(host=shard_address, port=shard_port)
        )