Repository: ray-project/ray

Issue:
[Core] Node Failure Long Running Test Failed at `grpc::ServerInterface::RegisteredAsyncRequest::IssueRequest()`
<!--Please include [tune], [rllib], [autoscaler] etc. in the issue title if relevant-->

### What is the problem?
Node Failure Long Running test fails with the error log below.

```
[2m[33m(pid=raylet)[0m E0429 02:32:06.263886 22036 process.cc:274] Failed to wait for process 22047 with error system:10: No child processes
E0429 02:32:12.346844 23272 task_manager.cc:288] 3 retries left for task b48f33dc1265b526ffffffff0100, attempting to resubmit.
E0429 02:32:12.346899 23272 core_worker.cc:373] Will resubmit task after a 5000ms delay: Type=NORMAL_TASK, Language=PYTHON, function_descriptor={type=PythonFunctionDescriptor, module_name=__main__, class_name=, function_name=f, function_hash=7d2c6c88e5e801d48a350076f2117e717fe12224}, task_id=b48f33dc1265b526ffffffff0100, job_id=0100, num_args=2, num_returns=1
[2m[33m(pid=raylet)[0m E0429 02:32:12.347446 22089 process.cc:274] Failed to wait for process 22100 with error system:10: No child processes
2020-04-29 02:32:12,653	INFO resource_spec.py:212 -- Starting Ray with 27.88 GiB memory available for workers and up to 0.15 GiB for objects. You can adjust these settings with ray.init(memory=<bytes>, object_store_memory=<bytes>).
[2m[33m(pid=raylet)[0m E0429 02:32:12.732946757   22142 server_chttp2.cc:40]        {"created":"@1588127532.732848116","description":"No address added out of total 1 resolved","file":"external/com_github_grpc_grpc/src/core/ext/transport/chttp2/server/chttp2_server.cc","file_line":394,"referenced_errors":[{"created":"@1588127532.732846227","description":"Failed to add any wildcard listeners","file":"external/com_github_grpc_grpc/src/core/lib/iomgr/tcp_server_posix.cc","file_line":341,"referenced_errors":[{"created":"@1588127532.732832876","description":"Unable to configure socket","fd":44,"file":"external/com_github_grpc_grpc/src/core/lib/iomgr/tcp_server_utils_posix_common.cc","file_line":208,"referenced_errors":[{"created":"@1588127532.732823689","description":"Address already in use","errno":98,"file":"external/com_github_grpc_grpc/src/core/lib/iomgr/tcp_server_utils_posix_common.cc","file_line":181,"os_error":"Address already in use","syscall":"bind"}]},{"created":"@1588127532.732845812","description":"Unable to configure socket","fd":44,"file":"external/com_github_grpc_grpc/src/core/lib/iomgr/tcp_server_utils_posix_common.cc","file_line":208,"referenced_errors":[{"created":"@1588127532.732843382","description":"Address already in use","errno":98,"file":"external/com_github_grpc_grpc/src/core/lib/iomgr/tcp_server_utils_posix_common.cc","file_line":181,"os_error":"Address already in use","syscall":"bind"}]}]}]}
[2m[33m(pid=raylet)[0m *** Aborted at 1588127532 (unix time) try "date -d @1588127532" if you are using GNU date ***
[2m[33m(pid=raylet)[0m PC: @                0x0 (unknown)
[2m[33m(pid=raylet)[0m *** SIGSEGV (@0x58) received by PID 22142 (TID 0x7fc3a66d37c0) from PID 88; stack trace: ***
[2m[33m(pid=raylet)[0m     @     0x7fc3a5c32390 (unknown)
[2m[33m(pid=raylet)[0m     @     0x5596e3957692 grpc::ServerInterface::RegisteredAsyncRequest::IssueRequest()
[2m[33m(pid=raylet)[0m     @     0x5596e35b2149 ray::rpc::NodeManagerService::WithAsyncMethod_RequestWorkerLease<>::RequestRequestWorkerLease()
[2m[33m(pid=raylet)[0m     @     0x5596e35c7b1b ray::rpc::ServerCallFactoryImpl<>::CreateCall()
[2m[33m(pid=raylet)[0m     @     0x5596e380bfe1 ray::rpc::GrpcServer::Run()
[2m[33m(pid=raylet)[0m     @     0x5596e3629acc ray::raylet::NodeManager::NodeManager()
[2m[33m(pid=raylet)[0m     @     0x5596e35cbc07 ray::raylet::Raylet::Raylet()
[2m[33m(pid=raylet)[0m     @     0x5596e359848d main
[2m[33m(pid=raylet)[0m     @     0x7fc3a5459830 __libc_start_main
[2m[33m(pid=raylet)[0m     @     0x5596e35a9391 (unknown)
[2m[36m(pid=22153)[0m E0429 02:32:13.864451 22153 raylet_client.cc:69] Retrying to connect to socket for pathname /tmp/ray/session_2020-04-28_20-19-44_770473_22870/sockets/raylet.8 (num_attempts = 1, num_retries = 5)
[2m[36m(pid=22153)[0m E0429 02:32:14.364712 22153 raylet_client.cc:69] Retrying to connect to socket for pathname /tmp/ray/session_2020-04-28_20-19-44_770473_22870/sockets/raylet.8 (num_attempts = 2, num_retries = 5)
[2m[36m(pid=22153)[0m E0429 02:32:14.864863 22153 raylet_client.cc:69] Retrying to connect to socket for pathname /tmp/ray/session_2020-04-28_20-19-44_770473_22870/sockets/raylet.8 (num_attempts = 3, num_retries = 5)
[2m[36m(pid=22153)[0m E0429 02:32:15.365000 22153 raylet_client.cc:69] Retrying to connect to socket for pathname /tmp/ray/session_2020-04-28_20-19-44_770473_22870/sockets/raylet.8 (num_attempts = 4, num_retries = 5)
[2m[36m(pid=22153)[0m F0429 02:32:15.865115 22153 raylet_client.cc:78] Could not connect to socket /tmp/ray/session_2020-04-28_20-19-44_770473_22870/sockets/raylet.8
[2m[36m(pid=22153)[0m *** Check failure stack trace: ***
[2m[36m(pid=22153)[0m     @     0x7f5d3b2b40ed  google::LogMessage::Fail()
[2m[36m(pid=22153)[0m     @     0x7f5d3b2b555c  google::LogMessage::SendToLog()
[2m[36m(pid=22153)[0m     @     0x7f5d3b2b3dc9  google::LogMessage::Flush()
[2m[36m(pid=22153)[0m     @     0x7f5d3b2b3fe1  google::LogMessage::~LogMessage()
[2m[36m(pid=22153)[0m     @     0x7f5d3b03bb39  ray::RayLog::~RayLog()
[2m[36m(pid=22153)[0m     @     0x7f5d3ae55133  ray::raylet::RayletConnection::RayletConnection()
[2m[36m(pid=22153)[0m     @     0x7f5d3ae55abf  ray::raylet::RayletClient::RayletClient()
[2m[36m(pid=22153)[0m     @     0x7f5d3adf513b  ray::CoreWorker::CoreWorker()
[2m[36m(pid=22153)[0m     @     0x7f5d3adf8984  ray::CoreWorkerProcess::CreateWorker()
[2m[36m(pid=22153)[0m     @     0x7f5d3adf8efb  ray::CoreWorkerProcess::CoreWorkerProcess()
[2m[36m(pid=22153)[0m     @     0x7f5d3adf93fb  ray::CoreWorkerProcess::Initialize()
[2m[36m(pid=22153)[0m     @     0x7f5d3ad6c06c  __pyx_pw_3ray_7_raylet_10CoreWorker_1__cinit__()
[2m[36m(pid=22153)[0m     @     0x7f5d3ad6d155  __pyx_tp_new_3ray_7_raylet_CoreWorker()
[2m[36m(pid=22153)[0m     @     0x55db24e47965  type_call
[2m[36m(pid=22153)[0m     @     0x55db24db7d7b  _PyObject_FastCallDict
[2m[36m(pid=22153)[0m     @     0x55db24e477ce  call_function
[2m[36m(pid=22153)[0m     @     0x55db24e69cba  _PyEval_EvalFrameDefault
[2m[36m(pid=22153)[0m     @     0x55db24e40dae  _PyEval_EvalCodeWithName
[2m[36m(pid=22153)[0m     @     0x55db24e41941  fast_function
[2m[36m(pid=22153)[0m     @     0x55db24e47755  call_function
[2m[36m(pid=22153)[0m     @     0x55db24e6aa7a  _PyEval_EvalFrameDefault
[2m[36m(pid=22153)[0m     @     0x55db24e42459  PyEval_EvalCodeEx
[2m[36m(pid=22153)[0m     @     0x55db24e431ec  PyEval_EvalCode
[2m[36m(pid=22153)[0m     @     0x55db24ebd9a4  run_mod
[2m[36m(pid=22153)[0m     @     0x55db24ebdda1  PyRun_FileExFlags
[2m[36m(pid=22153)[0m     @     0x55db24ebdfa4  PyRun_SimpleFileExFlags
[2m[36m(pid=22153)[0m     @     0x55db24ec1a9e  Py_Main
[2m[36m(pid=22153)[0m     @     0x55db24d894be  main
[2m[36m(pid=22153)[0m     @     0x7f5d3cd85830  __libc_start_main
[2m[36m(pid=22153)[0m     @     0x55db24e70773  (unknown)
Traceback (most recent call last):
  File "workloads/node_failures.py", line 57, in <module>
    cluster.add_node()
  File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ray/cluster_utils.py", line 115, in add_node
    self._wait_for_node(node)
  File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ray/cluster_utils.py", line 165, in _wait_for_node
    raise TimeoutError("Timed out while waiting for nodes to join.")
TimeoutError: Timed out while waiting for nodes to join.
[2m[33m(pid=raylet)[0m E0429 02:32:42.965368 13125 process.cc:274] Failed to wait for process 13136 with error system:10: No child processes
[2m[33m(pid=raylet)[0m E0429 02:32:43.045863  1167 process.cc:274] Failed to wait for process 1178 with error system:10: No child processes
2020-04-29 02:32:43,942	ERROR import_thread.py:93 -- ImportThread: Connection closed by server.
2020-04-29 02:32:43,942	ERROR worker.py:996 -- print_logs: Connection closed by server.
2020-04-29 02:32:43,942	ERROR worker.py:1096 -- listen_error_messages_raylet: Connection closed by server.
E0429 02:32:45.999132 22870 raylet_client.cc:90] IOError: [RayletClient] Connection closed unexpectedly. [RayletClient] Failed to disconnect from raylet.
```


*Ray version and other system information (Python version, TensorFlow version, OS):*

### Reproduction (REQUIRED)

This is hard to reproduce, but it happens consistently in multiple long running tests. For the logs above, it happens after 2909 iterations. 

- [ ] I have verified my script runs in a clean environment and reproduces the issue.
- [ ] I have verified the issue also occurs with the [latest wheels](https://docs.ray.io/en/latest/installation.html).



Discussion:
@stephanie-wang Are you working on this issue? 
I don't think this is a P0. It's just a side effect of rapidly killing and adding ray nodes on the same machine. Raylet gRPC server cannot bind to a port that's just released by killing ray node. 

This is uncommon in production scenarios. Moving it to P1. 

However, we do need to fix the long running test by adding some sleep buffer between killing and adding a new node, maybe inside test_reconstruction as well. 

It looks like the ultimate source of the problem is that the port for GPRC is selected before GRPC actually uses that port:
https://github.com/ray-project/ray/blob/137519e19d77debb5427a24279320f21e805867f/python/ray/node.py#L167


@simon-mo What are your thoughts on this?
One possible fix is to randomize the port instead of using kernel to get unused port. This can avoid possible race condition. The reason is that current `_get_unused_port` procedure returns unused port in sequence and is likely to duplicate port numbers, since we close the socket immediately and release the ownership of that port.
```python
    def _get_unused_port(self):
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.bind(("", 0))
        port = s.getsockname()[1]
        # fix: add some random integer to the port number, returns if available, else retry
        s.close()
        return port
```


Resolving patch:
diff --git a/python/ray/node.py b/python/ray/node.py
--- a/python/ray/node.py
+++ b/python/ray/node.py
@@ -4,6 +4,7 @@
 import errno
 import os
 import logging
+import random
 import signal
 import socket
 import subprocess
@@ -24,6 +25,7 @@
 logger = logging.getLogger(__name__)
 
 SESSION_LATEST = "session_latest"
+NUMBER_OF_PORT_RETRIES = 40
 
 
 class Node:
@@ -167,7 +169,8 @@ def __init__(self,
                 # NOTE: There is a possible but unlikely race condition where
                 # the port is bound by another process between now and when the
                 # raylet starts.
-                self._ray_params.node_manager_port = self._get_unused_port()
+                self._ray_params.node_manager_port, self._socket = \
+                    self._get_unused_port(close_on_exit=False)
 
         if not connect_only and spawn_reaper and not self.kernel_fate_share:
             self.start_reaper_process()
@@ -300,6 +303,14 @@ def node_manager_port(self):
         """Get the node manager's port."""
         return self._ray_params.node_manager_port
 
+    @property
+    def socket(self):
+        """Get the socket reserving the node manager's port"""
+        try:
+            return self._socket
+        except AttributeError:
+            return None
+
     @property
     def address_info(self):
         """Get a dictionary of addresses."""
@@ -395,12 +406,30 @@ def new_log_files(self, name):
         log_stderr_file = open(log_stderr, "a", buffering=1)
         return log_stdout_file, log_stderr_file
 
-    def _get_unused_port(self):
+    def _get_unused_port(self, close_on_exit=True):
         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         s.bind(("", 0))
         port = s.getsockname()[1]
-        s.close()
-        return port
+
+        # Try to generate a port that is far above the 'next available' one.
+        # This solves issue #8254 where GRPC fails because the port assigned
+        # from this method has been used by a different process.
+        for _ in range(NUMBER_OF_PORT_RETRIES):
+            new_port = random.randint(port, 65535)
+            new_s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            try:
+                new_s.bind(("", new_port))
+            except OSError:
+                new_s.close()
+                continue
+            s.close()
+            if close_on_exit:
+                new_s.close()
+            return new_port, new_s
+        logger.error("Unable to succeed in selecting a random port.")
+        if close_on_exit:
+            s.close()
+        return port, s
 
     def _prepare_socket_file(self, socket_path, default_prefix):
         """Prepare the socket file for raylet and plasma.
@@ -417,7 +446,7 @@ def _prepare_socket_file(self, socket_path, default_prefix):
         if sys.platform == "win32":
             if socket_path is None:
                 result = "tcp://{}:{}".format(self._localhost,
-                                              self._get_unused_port())
+                                              self._get_unused_port()[0])
         else:
             if socket_path is None:
                 result = self._make_inc_temp(
@@ -598,7 +627,8 @@ def start_raylet(self, use_valgrind=False, use_profiler=False):
             include_java=self._ray_params.include_java,
             java_worker_options=self._ray_params.java_worker_options,
             load_code_from_local=self._ray_params.load_code_from_local,
-            fate_share=self.kernel_fate_share)
+            fate_share=self.kernel_fate_share,
+            socket_to_use=self.socket)
         assert ray_constants.PROCESS_TYPE_RAYLET not in self.all_processes
         self.all_processes[ray_constants.PROCESS_TYPE_RAYLET] = [process_info]
 
diff --git a/python/ray/services.py b/python/ray/services.py
--- a/python/ray/services.py
+++ b/python/ray/services.py
@@ -1230,7 +1230,8 @@ def start_raylet(redis_address,
                  include_java=False,
                  java_worker_options=None,
                  load_code_from_local=False,
-                 fate_share=None):
+                 fate_share=None,
+                 socket_to_use=None):
     """Start a raylet, which is a combined local scheduler and object manager.
 
     Args:
@@ -1361,6 +1362,8 @@ def start_raylet(redis_address,
         "--temp_dir={}".format(temp_dir),
         "--session_dir={}".format(session_dir),
     ]
+    if socket_to_use:
+        socket_to_use.close()
     process_info = start_ray_process(
         command,
         ray_constants.PROCESS_TYPE_RAYLET,


Test patch: