# Buggy code
def _get_unused_port(self):
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind(("", 0))
    port = s.getsockname()[1]
    s.close()
    return port


# Traceback (TimeoutError)
�[2m�[33m(pid=raylet)�[0m E0429 02:32:06.263886 22036 process.cc:274] Failed to wait for process 22047 with error system:10: No child processes
E0429 02:32:12.346844 23272 task_manager.cc:288] 3 retries left for task b48f33dc1265b526ffffffff0100, attempting to resubmit.
E0429 02:32:12.346899 23272 core_worker.cc:373] Will resubmit task after a 5000ms delay: Type=NORMAL_TASK, Language=PYTHON, function_descriptor={type=PythonFunctionDescriptor, module_name=__main__, class_name=, function_name=f, function_hash=7d2c6c88e5e801d48a350076f2117e717fe12224}, task_id=b48f33dc1265b526ffffffff0100, job_id=0100, num_args=2, num_returns=1
�[2m�[33m(pid=raylet)�[0m E0429 02:32:12.347446 22089 process.cc:274] Failed to wait for process 22100 with error system:10: No child processes
2020-04-29 02:32:12,653	INFO resource_spec.py:212 -- Starting Ray with 27.88 GiB memory available for workers and up to 0.15 GiB for objects. You can adjust these settings with ray.init(memory=<bytes>, object_store_memory=<bytes>).
�[2m�[33m(pid=raylet)�[0m E0429 02:32:12.732946757   22142 server_chttp2.cc:40]        {"created":"@1588127532.732848116","description":"No address added out of total 1 resolved","file":"external/com_github_grpc_grpc/src/core/ext/transport/chttp2/server/chttp2_server.cc","file_line":394,"referenced_errors":[{"created":"@1588127532.732846227","description":"Failed to add any wildcard listeners","file":"external/com_github_grpc_grpc/src/core/lib/iomgr/tcp_server_posix.cc","file_line":341,"referenced_errors":[{"created":"@1588127532.732832876","description":"Unable to configure socket","fd":44,"file":"external/com_github_grpc_grpc/src/core/lib/iomgr/tcp_server_utils_posix_common.cc","file_line":208,"referenced_errors":[{"created":"@1588127532.732823689","description":"Address already in use","errno":98,"file":"external/com_github_grpc_grpc/src/core/lib/iomgr/tcp_server_utils_posix_common.cc","file_line":181,"os_error":"Address already in use","syscall":"bind"}]},{"created":"@1588127532.732845812","description":"Unable to configure socket","fd":44,"file":"external/com_github_grpc_grpc/src/core/lib/iomgr/tcp_server_utils_posix_common.cc","file_line":208,"referenced_errors":[{"created":"@1588127532.732843382","description":"Address already in use","errno":98,"file":"external/com_github_grpc_grpc/src/core/lib/iomgr/tcp_server_utils_posix_common.cc","file_line":181,"os_error":"Address already in use","syscall":"bind"}]}]}]}
�[2m�[33m(pid=raylet)�[0m *** Aborted at 1588127532 (unix time) try "date -d @1588127532" if you are using GNU date ***
�[2m�[33m(pid=raylet)�[0m PC: @                0x0 (unknown)
�[2m�[33m(pid=raylet)�[0m *** SIGSEGV (@0x58) received by PID 22142 (TID 0x7fc3a66d37c0) from PID 88; stack trace: ***
�[2m�[33m(pid=raylet)�[0m     @     0x7fc3a5c32390 (unknown)
�[2m�[33m(pid=raylet)�[0m     @     0x5596e3957692 grpc::ServerInterface::RegisteredAsyncRequest::IssueRequest()
�[2m�[33m(pid=raylet)�[0m     @     0x5596e35b2149 ray::rpc::NodeManagerService::WithAsyncMethod_RequestWorkerLease<>::RequestRequestWorkerLease()
�[2m�[33m(pid=raylet)�[0m     @     0x5596e35c7b1b ray::rpc::ServerCallFactoryImpl<>::CreateCall()
�[2m�[33m(pid=raylet)�[0m     @     0x5596e380bfe1 ray::rpc::GrpcServer::Run()
�[2m�[33m(pid=raylet)�[0m     @     0x5596e3629acc ray::raylet::NodeManager::NodeManager()
�[2m�[33m(pid=raylet)�[0m     @     0x5596e35cbc07 ray::raylet::Raylet::Raylet()
�[2m�[33m(pid=raylet)�[0m     @     0x5596e359848d main
�[2m�[33m(pid=raylet)�[0m     @     0x7fc3a5459830 __libc_start_main
�[2m�[33m(pid=raylet)�[0m     @     0x5596e35a9391 (unknown)
�[2m�[36m(pid=22153)�[0m E0429 02:32:13.864451 22153 raylet_client.cc:69] Retrying to connect to socket for pathname /tmp/ray/session_2020-04-28_20-19-44_770473_22870/sockets/raylet.8 (num_attempts = 1, num_retries = 5)
�[2m�[36m(pid=22153)�[0m E0429 02:32:14.364712 22153 raylet_client.cc:69] Retrying to connect to socket for pathname /tmp/ray/session_2020-04-28_20-19-44_770473_22870/sockets/raylet.8 (num_attempts = 2, num_retries = 5)
�[2m�[36m(pid=22153)�[0m E0429 02:32:14.864863 22153 raylet_client.cc:69] Retrying to connect to socket for pathname /tmp/ray/session_2020-04-28_20-19-44_770473_22870/sockets/raylet.8 (num_attempts = 3, num_retries = 5)
�[2m�[36m(pid=22153)�[0m E0429 02:32:15.365000 22153 raylet_client.cc:69] Retrying to connect to socket for pathname /tmp/ray/session_2020-04-28_20-19-44_770473_22870/sockets/raylet.8 (num_attempts = 4, num_retries = 5)
�[2m�[36m(pid=22153)�[0m F0429 02:32:15.865115 22153 raylet_client.cc:78] Could not connect to socket /tmp/ray/session_2020-04-28_20-19-44_770473_22870/sockets/raylet.8
�[2m�[36m(pid=22153)�[0m *** Check failure stack trace: ***
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3b2b40ed  google::LogMessage::Fail()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3b2b555c  google::LogMessage::SendToLog()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3b2b3dc9  google::LogMessage::Flush()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3b2b3fe1  google::LogMessage::~LogMessage()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3b03bb39  ray::RayLog::~RayLog()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3ae55133  ray::raylet::RayletConnection::RayletConnection()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3ae55abf  ray::raylet::RayletClient::RayletClient()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3adf513b  ray::CoreWorker::CoreWorker()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3adf8984  ray::CoreWorkerProcess::CreateWorker()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3adf8efb  ray::CoreWorkerProcess::CoreWorkerProcess()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3adf93fb  ray::CoreWorkerProcess::Initialize()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3ad6c06c  __pyx_pw_3ray_7_raylet_10CoreWorker_1__cinit__()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3ad6d155  __pyx_tp_new_3ray_7_raylet_CoreWorker()
�[2m�[36m(pid=22153)�[0m     @     0x55db24e47965  type_call
�[2m�[36m(pid=22153)�[0m     @     0x55db24db7d7b  _PyObject_FastCallDict
�[2m�[36m(pid=22153)�[0m     @     0x55db24e477ce  call_function
�[2m�[36m(pid=22153)�[0m     @     0x55db24e69cba  _PyEval_EvalFrameDefault
�[2m�[36m(pid=22153)�[0m     @     0x55db24e40dae  _PyEval_EvalCodeWithName
�[2m�[36m(pid=22153)�[0m     @     0x55db24e41941  fast_function
�[2m�[36m(pid=22153)�[0m     @     0x55db24e47755  call_function
�[2m�[36m(pid=22153)�[0m     @     0x55db24e6aa7a  _PyEval_EvalFrameDefault
�[2m�[36m(pid=22153)�[0m     @     0x55db24e42459  PyEval_EvalCodeEx
�[2m�[36m(pid=22153)�[0m     @     0x55db24e431ec  PyEval_EvalCode
�[2m�[36m(pid=22153)�[0m     @     0x55db24ebd9a4  run_mod
�[2m�[36m(pid=22153)�[0m     @     0x55db24ebdda1  PyRun_FileExFlags
�[2m�[36m(pid=22153)�[0m     @     0x55db24ebdfa4  PyRun_SimpleFileExFlags
�[2m�[36m(pid=22153)�[0m     @     0x55db24ec1a9e  Py_Main
�[2m�[36m(pid=22153)�[0m     @     0x55db24d894be  main
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3cd85830  __libc_start_main
�[2m�[36m(pid=22153)�[0m     @     0x55db24e70773  (unknown)
Traceback (most recent call last):
File "workloads/node_failures.py", line 57, in <module>
cluster.add_node()
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ray/cluster_utils.py", line 115, in add_node
self._wait_for_node(node)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ray/cluster_utils.py", line 165, in _wait_for_node
raise TimeoutError("Timed out while waiting for nodes to join.")
TimeoutError: Timed out while waiting for nodes to join.
�[2m�[33m(pid=raylet)�[0m E0429 02:32:42.965368 13125 process.cc:274] Failed to wait for process 13136 with error system:10: No child processes
�[2m�[33m(pid=raylet)�[0m E0429 02:32:43.045863  1167 process.cc:274] Failed to wait for process 1178 with error system:10: No child processes
2020-04-29 02:32:43,942	ERROR import_thread.py:93 -- ImportThread: Connection closed by server.
2020-04-29 02:32:43,942	ERROR worker.py:996 -- print_logs: Connection closed by server.
2020-04-29 02:32:43,942	ERROR worker.py:1096 -- listen_error_messages_raylet: Connection closed by server.
E0429 02:32:45.999132 22870 raylet_client.cc:90] IOError: [RayletClient] Connection closed unexpectedly. [RayletClient] Failed to disconnect from raylet.

# Fixed code
def _get_unused_port(self, close_on_exit=True):
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind(("", 0))
    port = s.getsockname()[1]

    # Try to generate a port that is far above the 'next available' one.
    # This solves issue #8254 where GRPC fails because the port assigned
    # from this method has been used by a different process.
    for _ in range(NUMBER_OF_PORT_RETRIES):
        new_port = random.randint(port, 65535)
        new_s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            new_s.bind(("", new_port))
        except OSError:
            new_s.close()
            continue
        s.close()
        if close_on_exit:
            new_s.close()
        return new_port, new_s
    logger.error("Unable to succeed in selecting a random port.")
    if close_on_exit:
        s.close()
    return port, s