# Buggy code
def __init__(
    self,
    ray_params,
    head=False,
    shutdown_at_exit=True,
    spawn_reaper=True,
    connect_only=False,
):
    """Start a node.

    Args:
        ray_params (ray.params.RayParams): The parameters to use to
            configure the node.
        head (bool): True if this is the head node, which means it will
            start additional processes like the Redis servers, monitor
            processes, and web UI.
        shutdown_at_exit (bool): If true, spawned processes will be cleaned
            up if this process exits normally.
        spawn_reaper (bool): If true, spawns a process that will clean up
            other spawned processes if this process dies unexpectedly.
        connect_only (bool): If true, connect to the node without starting
            new processes.
    """
    if shutdown_at_exit:
        if connect_only:
            raise ValueError(
                "'shutdown_at_exit' and 'connect_only' cannot both be true."
            )
        self._register_shutdown_hooks()

    self.head = head
    self.kernel_fate_share = bool(
        spawn_reaper and ray.utils.detect_fate_sharing_support()
    )
    self.all_processes = {}

    # Try to get node IP address with the parameters.
    if ray_params.node_ip_address:
        node_ip_address = ray_params.node_ip_address
    elif ray_params.redis_address:
        node_ip_address = ray.services.get_node_ip_address(ray_params.redis_address)
    else:
        node_ip_address = ray.services.get_node_ip_address()
    self._node_ip_address = node_ip_address

    if ray_params.raylet_ip_address:
        raylet_ip_address = ray_params.raylet_ip_address
    else:
        raylet_ip_address = node_ip_address

    if raylet_ip_address != node_ip_address and (not connect_only or head):
        raise ValueError(
            "The raylet IP address should only be different than the node "
            "IP address when connecting to an existing raylet; i.e., when "
            "head=False and connect_only=True."
        )

    self._raylet_ip_address = raylet_ip_address

    ray_params.update_if_absent(
        include_log_monitor=True,
        resources={},
        temp_dir=ray.utils.get_ray_temp_dir(),
        worker_path=os.path.join(
            os.path.dirname(os.path.abspath(__file__)), "workers/default_worker.py"
        ),
    )

    self._resource_spec = None
    self._localhost = socket.gethostbyname("localhost")
    self._ray_params = ray_params
    self._redis_address = ray_params.redis_address
    self._config = ray_params._internal_config

    if head:
        redis_client = None
        # date including microsecond
        date_str = datetime.datetime.today().strftime("%Y-%m-%d_%H-%M-%S_%f")
        self.session_name = "session_{date_str}_{pid}".format(
            pid=os.getpid(), date_str=date_str
        )
    else:
        redis_client = self.create_redis_client()
        self.session_name = ray.utils.decode(redis_client.get("session_name"))

    self._init_temp(redis_client)

    if connect_only:
        # Get socket names from the configuration.
        self._plasma_store_socket_name = ray_params.plasma_store_socket_name
        self._raylet_socket_name = ray_params.raylet_socket_name

        # If user does not provide the socket name, get it from Redis.
        if (
            self._plasma_store_socket_name is None
            or self._raylet_socket_name is None
            or self._ray_params.node_manager_port is None
        ):
            # Get the address info of the processes to connect to
            # from Redis.
            address_info = ray.services.get_address_info_from_redis(
                self.redis_address,
                self._raylet_ip_address,
                redis_password=self.redis_password,
            )
            self._plasma_store_socket_name = address_info["object_store_address"]
            self._raylet_socket_name = address_info["raylet_socket_name"]
            self._ray_params.node_manager_port = address_info["node_manager_port"]
    else:
        # If the user specified a socket name, use it.
        self._plasma_store_socket_name = self._prepare_socket_file(
            self._ray_params.plasma_store_socket_name, default_prefix="plasma_store"
        )
        self._raylet_socket_name = self._prepare_socket_file(
            self._ray_params.raylet_socket_name, default_prefix="raylet"
        )

    if head:
        ray_params.update_if_absent(num_redis_shards=1)
        self._webui_url = None
    else:
        self._webui_url = ray.services.get_webui_url_from_redis(redis_client)
        ray_params.include_java = ray.services.include_java_from_redis(redis_client)

    if head or not connect_only:
        # We need to start a local raylet.
        if (
            self._ray_params.node_manager_port is None
            or self._ray_params.node_manager_port == 0
        ):
            # No port specified. Pick a random port for the raylet to use.
            # NOTE: There is a possible but unlikely race condition where
            # the port is bound by another process between now and when the
            # raylet starts.
            self._ray_params.node_manager_port = self._get_unused_port()

    if not connect_only and spawn_reaper and not self.kernel_fate_share:
        self.start_reaper_process()

    # Start processes.
    if head:
        self.start_head_processes()
        redis_client = self.create_redis_client()
        redis_client.set("session_name", self.session_name)
        redis_client.set("session_dir", self._session_dir)
        redis_client.set("temp_dir", self._temp_dir)

    if not connect_only:
        self.start_ray_processes()


# Traceback (TimeoutError)
�[2m�[33m(pid=raylet)�[0m E0429 02:32:06.263886 22036 process.cc:274] Failed to wait for process 22047 with error system:10: No child processes
E0429 02:32:12.346844 23272 task_manager.cc:288] 3 retries left for task b48f33dc1265b526ffffffff0100, attempting to resubmit.
E0429 02:32:12.346899 23272 core_worker.cc:373] Will resubmit task after a 5000ms delay: Type=NORMAL_TASK, Language=PYTHON, function_descriptor={type=PythonFunctionDescriptor, module_name=__main__, class_name=, function_name=f, function_hash=7d2c6c88e5e801d48a350076f2117e717fe12224}, task_id=b48f33dc1265b526ffffffff0100, job_id=0100, num_args=2, num_returns=1
�[2m�[33m(pid=raylet)�[0m E0429 02:32:12.347446 22089 process.cc:274] Failed to wait for process 22100 with error system:10: No child processes
2020-04-29 02:32:12,653	INFO resource_spec.py:212 -- Starting Ray with 27.88 GiB memory available for workers and up to 0.15 GiB for objects. You can adjust these settings with ray.init(memory=<bytes>, object_store_memory=<bytes>).
�[2m�[33m(pid=raylet)�[0m E0429 02:32:12.732946757   22142 server_chttp2.cc:40]        {"created":"@1588127532.732848116","description":"No address added out of total 1 resolved","file":"external/com_github_grpc_grpc/src/core/ext/transport/chttp2/server/chttp2_server.cc","file_line":394,"referenced_errors":[{"created":"@1588127532.732846227","description":"Failed to add any wildcard listeners","file":"external/com_github_grpc_grpc/src/core/lib/iomgr/tcp_server_posix.cc","file_line":341,"referenced_errors":[{"created":"@1588127532.732832876","description":"Unable to configure socket","fd":44,"file":"external/com_github_grpc_grpc/src/core/lib/iomgr/tcp_server_utils_posix_common.cc","file_line":208,"referenced_errors":[{"created":"@1588127532.732823689","description":"Address already in use","errno":98,"file":"external/com_github_grpc_grpc/src/core/lib/iomgr/tcp_server_utils_posix_common.cc","file_line":181,"os_error":"Address already in use","syscall":"bind"}]},{"created":"@1588127532.732845812","description":"Unable to configure socket","fd":44,"file":"external/com_github_grpc_grpc/src/core/lib/iomgr/tcp_server_utils_posix_common.cc","file_line":208,"referenced_errors":[{"created":"@1588127532.732843382","description":"Address already in use","errno":98,"file":"external/com_github_grpc_grpc/src/core/lib/iomgr/tcp_server_utils_posix_common.cc","file_line":181,"os_error":"Address already in use","syscall":"bind"}]}]}]}
�[2m�[33m(pid=raylet)�[0m *** Aborted at 1588127532 (unix time) try "date -d @1588127532" if you are using GNU date ***
�[2m�[33m(pid=raylet)�[0m PC: @                0x0 (unknown)
�[2m�[33m(pid=raylet)�[0m *** SIGSEGV (@0x58) received by PID 22142 (TID 0x7fc3a66d37c0) from PID 88; stack trace: ***
�[2m�[33m(pid=raylet)�[0m     @     0x7fc3a5c32390 (unknown)
�[2m�[33m(pid=raylet)�[0m     @     0x5596e3957692 grpc::ServerInterface::RegisteredAsyncRequest::IssueRequest()
�[2m�[33m(pid=raylet)�[0m     @     0x5596e35b2149 ray::rpc::NodeManagerService::WithAsyncMethod_RequestWorkerLease<>::RequestRequestWorkerLease()
�[2m�[33m(pid=raylet)�[0m     @     0x5596e35c7b1b ray::rpc::ServerCallFactoryImpl<>::CreateCall()
�[2m�[33m(pid=raylet)�[0m     @     0x5596e380bfe1 ray::rpc::GrpcServer::Run()
�[2m�[33m(pid=raylet)�[0m     @     0x5596e3629acc ray::raylet::NodeManager::NodeManager()
�[2m�[33m(pid=raylet)�[0m     @     0x5596e35cbc07 ray::raylet::Raylet::Raylet()
�[2m�[33m(pid=raylet)�[0m     @     0x5596e359848d main
�[2m�[33m(pid=raylet)�[0m     @     0x7fc3a5459830 __libc_start_main
�[2m�[33m(pid=raylet)�[0m     @     0x5596e35a9391 (unknown)
�[2m�[36m(pid=22153)�[0m E0429 02:32:13.864451 22153 raylet_client.cc:69] Retrying to connect to socket for pathname /tmp/ray/session_2020-04-28_20-19-44_770473_22870/sockets/raylet.8 (num_attempts = 1, num_retries = 5)
�[2m�[36m(pid=22153)�[0m E0429 02:32:14.364712 22153 raylet_client.cc:69] Retrying to connect to socket for pathname /tmp/ray/session_2020-04-28_20-19-44_770473_22870/sockets/raylet.8 (num_attempts = 2, num_retries = 5)
�[2m�[36m(pid=22153)�[0m E0429 02:32:14.864863 22153 raylet_client.cc:69] Retrying to connect to socket for pathname /tmp/ray/session_2020-04-28_20-19-44_770473_22870/sockets/raylet.8 (num_attempts = 3, num_retries = 5)
�[2m�[36m(pid=22153)�[0m E0429 02:32:15.365000 22153 raylet_client.cc:69] Retrying to connect to socket for pathname /tmp/ray/session_2020-04-28_20-19-44_770473_22870/sockets/raylet.8 (num_attempts = 4, num_retries = 5)
�[2m�[36m(pid=22153)�[0m F0429 02:32:15.865115 22153 raylet_client.cc:78] Could not connect to socket /tmp/ray/session_2020-04-28_20-19-44_770473_22870/sockets/raylet.8
�[2m�[36m(pid=22153)�[0m *** Check failure stack trace: ***
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3b2b40ed  google::LogMessage::Fail()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3b2b555c  google::LogMessage::SendToLog()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3b2b3dc9  google::LogMessage::Flush()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3b2b3fe1  google::LogMessage::~LogMessage()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3b03bb39  ray::RayLog::~RayLog()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3ae55133  ray::raylet::RayletConnection::RayletConnection()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3ae55abf  ray::raylet::RayletClient::RayletClient()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3adf513b  ray::CoreWorker::CoreWorker()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3adf8984  ray::CoreWorkerProcess::CreateWorker()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3adf8efb  ray::CoreWorkerProcess::CoreWorkerProcess()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3adf93fb  ray::CoreWorkerProcess::Initialize()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3ad6c06c  __pyx_pw_3ray_7_raylet_10CoreWorker_1__cinit__()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3ad6d155  __pyx_tp_new_3ray_7_raylet_CoreWorker()
�[2m�[36m(pid=22153)�[0m     @     0x55db24e47965  type_call
�[2m�[36m(pid=22153)�[0m     @     0x55db24db7d7b  _PyObject_FastCallDict
�[2m�[36m(pid=22153)�[0m     @     0x55db24e477ce  call_function
�[2m�[36m(pid=22153)�[0m     @     0x55db24e69cba  _PyEval_EvalFrameDefault
�[2m�[36m(pid=22153)�[0m     @     0x55db24e40dae  _PyEval_EvalCodeWithName
�[2m�[36m(pid=22153)�[0m     @     0x55db24e41941  fast_function
�[2m�[36m(pid=22153)�[0m     @     0x55db24e47755  call_function
�[2m�[36m(pid=22153)�[0m     @     0x55db24e6aa7a  _PyEval_EvalFrameDefault
�[2m�[36m(pid=22153)�[0m     @     0x55db24e42459  PyEval_EvalCodeEx
�[2m�[36m(pid=22153)�[0m     @     0x55db24e431ec  PyEval_EvalCode
�[2m�[36m(pid=22153)�[0m     @     0x55db24ebd9a4  run_mod
�[2m�[36m(pid=22153)�[0m     @     0x55db24ebdda1  PyRun_FileExFlags
�[2m�[36m(pid=22153)�[0m     @     0x55db24ebdfa4  PyRun_SimpleFileExFlags
�[2m�[36m(pid=22153)�[0m     @     0x55db24ec1a9e  Py_Main
�[2m�[36m(pid=22153)�[0m     @     0x55db24d894be  main
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3cd85830  __libc_start_main
�[2m�[36m(pid=22153)�[0m     @     0x55db24e70773  (unknown)
Traceback (most recent call last):
File "workloads/node_failures.py", line 57, in <module>
cluster.add_node()
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ray/cluster_utils.py", line 115, in add_node
self._wait_for_node(node)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ray/cluster_utils.py", line 165, in _wait_for_node
raise TimeoutError("Timed out while waiting for nodes to join.")
TimeoutError: Timed out while waiting for nodes to join.
�[2m�[33m(pid=raylet)�[0m E0429 02:32:42.965368 13125 process.cc:274] Failed to wait for process 13136 with error system:10: No child processes
�[2m�[33m(pid=raylet)�[0m E0429 02:32:43.045863  1167 process.cc:274] Failed to wait for process 1178 with error system:10: No child processes
2020-04-29 02:32:43,942	ERROR import_thread.py:93 -- ImportThread: Connection closed by server.
2020-04-29 02:32:43,942	ERROR worker.py:996 -- print_logs: Connection closed by server.
2020-04-29 02:32:43,942	ERROR worker.py:1096 -- listen_error_messages_raylet: Connection closed by server.
E0429 02:32:45.999132 22870 raylet_client.cc:90] IOError: [RayletClient] Connection closed unexpectedly. [RayletClient] Failed to disconnect from raylet.

# Fixed code
def __init__(
    self,
    ray_params,
    head=False,
    shutdown_at_exit=True,
    spawn_reaper=True,
    connect_only=False,
):
    """Start a node.

    Args:
        ray_params (ray.params.RayParams): The parameters to use to
            configure the node.
        head (bool): True if this is the head node, which means it will
            start additional processes like the Redis servers, monitor
            processes, and web UI.
        shutdown_at_exit (bool): If true, spawned processes will be cleaned
            up if this process exits normally.
        spawn_reaper (bool): If true, spawns a process that will clean up
            other spawned processes if this process dies unexpectedly.
        connect_only (bool): If true, connect to the node without starting
            new processes.
    """
    if shutdown_at_exit:
        if connect_only:
            raise ValueError(
                "'shutdown_at_exit' and 'connect_only' cannot both be true."
            )
        self._register_shutdown_hooks()

    self.head = head
    self.kernel_fate_share = bool(
        spawn_reaper and ray.utils.detect_fate_sharing_support()
    )
    self.all_processes = {}

    # Try to get node IP address with the parameters.
    if ray_params.node_ip_address:
        node_ip_address = ray_params.node_ip_address
    elif ray_params.redis_address:
        node_ip_address = ray.services.get_node_ip_address(ray_params.redis_address)
    else:
        node_ip_address = ray.services.get_node_ip_address()
    self._node_ip_address = node_ip_address

    if ray_params.raylet_ip_address:
        raylet_ip_address = ray_params.raylet_ip_address
    else:
        raylet_ip_address = node_ip_address

    if raylet_ip_address != node_ip_address and (not connect_only or head):
        raise ValueError(
            "The raylet IP address should only be different than the node "
            "IP address when connecting to an existing raylet; i.e., when "
            "head=False and connect_only=True."
        )

    self._raylet_ip_address = raylet_ip_address

    ray_params.update_if_absent(
        include_log_monitor=True,
        resources={},
        temp_dir=ray.utils.get_ray_temp_dir(),
        worker_path=os.path.join(
            os.path.dirname(os.path.abspath(__file__)), "workers/default_worker.py"
        ),
    )

    self._resource_spec = None
    self._localhost = socket.gethostbyname("localhost")
    self._ray_params = ray_params
    self._redis_address = ray_params.redis_address
    self._config = ray_params._internal_config

    if head:
        redis_client = None
        # date including microsecond
        date_str = datetime.datetime.today().strftime("%Y-%m-%d_%H-%M-%S_%f")
        self.session_name = "session_{date_str}_{pid}".format(
            pid=os.getpid(), date_str=date_str
        )
    else:
        redis_client = self.create_redis_client()
        self.session_name = ray.utils.decode(redis_client.get("session_name"))

    self._init_temp(redis_client)

    if connect_only:
        # Get socket names from the configuration.
        self._plasma_store_socket_name = ray_params.plasma_store_socket_name
        self._raylet_socket_name = ray_params.raylet_socket_name

        # If user does not provide the socket name, get it from Redis.
        if (
            self._plasma_store_socket_name is None
            or self._raylet_socket_name is None
            or self._ray_params.node_manager_port is None
        ):
            # Get the address info of the processes to connect to
            # from Redis.
            address_info = ray.services.get_address_info_from_redis(
                self.redis_address,
                self._raylet_ip_address,
                redis_password=self.redis_password,
            )
            self._plasma_store_socket_name = address_info["object_store_address"]
            self._raylet_socket_name = address_info["raylet_socket_name"]
            self._ray_params.node_manager_port = address_info["node_manager_port"]
    else:
        # If the user specified a socket name, use it.
        self._plasma_store_socket_name = self._prepare_socket_file(
            self._ray_params.plasma_store_socket_name, default_prefix="plasma_store"
        )
        self._raylet_socket_name = self._prepare_socket_file(
            self._ray_params.raylet_socket_name, default_prefix="raylet"
        )

    if head:
        ray_params.update_if_absent(num_redis_shards=1)
        self._webui_url = None
    else:
        self._webui_url = ray.services.get_webui_url_from_redis(redis_client)
        ray_params.include_java = ray.services.include_java_from_redis(redis_client)

    if head or not connect_only:
        # We need to start a local raylet.
        if (
            self._ray_params.node_manager_port is None
            or self._ray_params.node_manager_port == 0
        ):
            # No port specified. Pick a random port for the raylet to use.
            # NOTE: There is a possible but unlikely race condition where
            # the port is bound by another process between now and when the
            # raylet starts.
            self._ray_params.node_manager_port, self._socket = self._get_unused_port(
                close_on_exit=False
            )

    if not connect_only and spawn_reaper and not self.kernel_fate_share:
        self.start_reaper_process()

    # Start processes.
    if head:
        self.start_head_processes()
        redis_client = self.create_redis_client()
        redis_client.set("session_name", self.session_name)
        redis_client.set("session_dir", self._session_dir)
        redis_client.set("temp_dir", self._temp_dir)

    if not connect_only:
        self.start_ray_processes()