Repository: ray-project/ray

Issue:
No jobs scheduled when using redis-password
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: Ubuntu 16.04.1
- **Ray installed from (source or binary)**: pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.5.3-cp35-cp35m-manylinux1_x86_64.whl
- **Ray version**: 0.5.3
- **Python version**: 3.5.2
- **Exact command to reproduce**: Given `foo.py` below, running the following reproduces the problem:
```
export RAY_USE_XRAY=1
ray start --head --redis-password=thequickbrownfox
python foo.py
```

### Describe the problem
When I start ray without a `--redis-password`, `foo.py` works as expected (writes 5 `foo#.out` files). When using the `--redis-password`, `foo.py` hangs indefinitely.

### Source code / logs
`foo.py`:
```
import ray
# using whatever address and port were given when I ran ray start
ray.init(redis_address="<addr>:<port>", redis_password="thequickbrownfox")
@ray.remote
def foo(i):
    with open("foo{}.out".format(i), "w") as fooF:
        fooF.write("hello world")

ids = []
for i in range(5):
    ids.append(foo.remote(i))

ray.get(ids)
print("done!")
```


Discussion:

Resolving patch:
diff --git a/python/ray/services.py b/python/ray/services.py
--- a/python/ray/services.py
+++ b/python/ray/services.py
@@ -1216,6 +1216,8 @@ def start_raylet_monitor(redis_address,
     gcs_ip_address, gcs_port = redis_address.split(":")
     redis_password = redis_password or ""
     command = [RAYLET_MONITOR_EXECUTABLE, gcs_ip_address, gcs_port]
+    if redis_password:
+        command += [redis_password]
     p = subprocess.Popen(command, stdout=stdout_file, stderr=stderr_file)
     if cleanup:
         all_processes[PROCESS_TYPE_MONITOR].append(p)
diff --git a/python/ray/worker.py b/python/ray/worker.py
--- a/python/ray/worker.py
+++ b/python/ray/worker.py
@@ -1933,8 +1933,9 @@ def connect(info,
             sys.stdout = log_stdout_file
             sys.stderr = log_stderr_file
             services.record_log_files_in_redis(
-                info["redis_address"], info["node_ip_address"],
-                [log_stdout_file, log_stderr_file])
+                info["redis_address"],
+                info["node_ip_address"], [log_stdout_file, log_stderr_file],
+                password=redis_password)
 
     # Create an object for interfacing with the global state.
     global_state._initialize_global_state(


Test patch:
diff --git a/python/ray/test/cluster_utils.py b/python/ray/test/cluster_utils.py
--- a/python/ray/test/cluster_utils.py
+++ b/python/ray/test/cluster_utils.py
@@ -37,7 +37,10 @@ def __init__(self,
             head_node_args = head_node_args or {}
             self.add_node(**head_node_args)
             if connect:
-                ray.init(redis_address=self.redis_address)
+                redis_password = head_node_args.get("redis_password")
+                ray.init(
+                    redis_address=self.redis_address,
+                    redis_password=redis_password)
 
     def add_node(self, **override_kwargs):
         """Adds a node to the local Ray Cluster.
diff --git a/python/ray/test/test_ray_init.py b/python/ray/test/test_ray_init.py
--- a/python/ray/test/test_ray_init.py
+++ b/python/ray/test/test_ray_init.py
@@ -7,6 +7,7 @@
 import redis
 
 import ray
+from ray.test.cluster_utils import Cluster
 
 
 @pytest.fixture
@@ -29,7 +30,6 @@ class TestRedisPassword(object):
         os.environ.get("RAY_USE_NEW_GCS") == "on",
         reason="New GCS API doesn't support Redis authentication yet.")
     def test_redis_password(self, password, shutdown_only):
-        # Workaround for https://github.com/ray-project/ray/issues/3045
         @ray.remote
         def f():
             return 1
@@ -52,3 +52,19 @@ def f():
         redis_client = redis.StrictRedis(
             host=redis_ip, port=redis_port, password=password)
         assert redis_client.ping()
+
+    @pytest.mark.skipif(
+        os.environ.get("RAY_USE_NEW_GCS") == "on",
+        reason="New GCS API doesn't support Redis authentication yet.")
+    def test_redis_password_cluster(self, password, shutdown_only):
+        @ray.remote
+        def f():
+            return 1
+
+        node_args = {"redis_password": password}
+        cluster = Cluster(
+            initialize_head=True, connect=True, head_node_args=node_args)
+        cluster.add_node(**node_args)
+
+        object_id = f.remote()
+        ray.get(object_id)