# Buggy code
def start_raylet(
    redis_address,
    node_ip_address,
    node_manager_port,
    raylet_name,
    plasma_store_name,
    worker_path,
    temp_dir,
    session_dir,
    resource_spec,
    min_worker_port=None,
    max_worker_port=None,
    object_manager_port=None,
    redis_password=None,
    use_valgrind=False,
    use_profiler=False,
    stdout_file=None,
    stderr_file=None,
    config=None,
    include_java=False,
    java_worker_options=None,
    load_code_from_local=False,
    fate_share=None,
):
    """Start a raylet, which is a combined local scheduler and object manager.

    Args:
        redis_address (str): The address of the primary Redis server.
        node_ip_address (str): The IP address of this node.
        node_manager_port(int): The port to use for the node manager. This must
            not be 0.
        raylet_name (str): The name of the raylet socket to create.
        plasma_store_name (str): The name of the plasma store socket to connect
             to.
        worker_path (str): The path of the Python file that new worker
            processes will execute.
        temp_dir (str): The path of the temporary directory Ray will use.
        session_dir (str): The path of this session.
        resource_spec (ResourceSpec): Resources for this raylet.
        object_manager_port: The port to use for the object manager. If this is
            None, then the object manager will choose its own port.
        min_worker_port (int): The lowest port number that workers will bind
            on. If not set, random ports will be chosen.
        max_worker_port (int): The highest port number that workers will bind
            on. If set, min_worker_port must also be set.
        redis_password: The password to use when connecting to Redis.
        use_valgrind (bool): True if the raylet should be started inside
            of valgrind. If this is True, use_profiler must be False.
        use_profiler (bool): True if the raylet should be started inside
            a profiler. If this is True, use_valgrind must be False.
        stdout_file: A file handle opened for writing to redirect stdout to. If
            no redirection should happen, then this should be None.
        stderr_file: A file handle opened for writing to redirect stderr to. If
            no redirection should happen, then this should be None.
        config (dict|None): Optional Raylet configuration that will
            override defaults in RayConfig.
        include_java (bool): If True, the raylet backend can also support
            Java worker.
        java_worker_options (list): The command options for Java worker.
    Returns:
        ProcessInfo for the process that was started.
    """
    # The caller must provide a node manager port so that we can correctly
    # populate the command to start a worker.
    assert node_manager_port is not None and node_manager_port != 0
    config = config or {}
    config_str = ",".join(["{},{}".format(*kv) for kv in config.items()])

    if use_valgrind and use_profiler:
        raise ValueError("Cannot use valgrind and profiler at the same time.")

    assert resource_spec.resolved()
    num_initial_workers = resource_spec.num_cpus
    static_resources = resource_spec.to_resource_dict()

    # Limit the number of workers that can be started in parallel by the
    # raylet. However, make sure it is at least 1.
    num_cpus_static = static_resources.get("CPU", 0)
    maximum_startup_concurrency = max(
        1, min(multiprocessing.cpu_count(), num_cpus_static)
    )

    # Format the resource argument in a form like 'CPU,1.0,GPU,0,Custom,3'.
    resource_argument = ",".join(
        ["{},{}".format(*kv) for kv in static_resources.items()]
    )

    gcs_ip_address, gcs_port = redis_address.split(":")

    if include_java is True:
        default_cp = os.pathsep.join(DEFAULT_JAVA_WORKER_CLASSPATH)
        java_worker_command = build_java_worker_command(
            json.loads(java_worker_options)
            if java_worker_options
            else ["-classpath", default_cp],
            redis_address,
            node_manager_port,
            plasma_store_name,
            raylet_name,
            redis_password,
            session_dir,
        )
    else:
        java_worker_command = []

    # Create the command that the Raylet will use to start workers.
    start_worker_command = [
        sys.executable,
        worker_path,
        "--node-ip-address={}".format(node_ip_address),
        "--node-manager-port={}".format(node_manager_port),
        "--object-store-name={}".format(plasma_store_name),
        "--raylet-name={}".format(raylet_name),
        "--redis-address={}".format(redis_address),
        "--config-list={}".format(config_str),
        "--temp-dir={}".format(temp_dir),
    ]
    if redis_password:
        start_worker_command += ["--redis-password={}".format(redis_password)]

    # If the object manager port is None, then use 0 to cause the object
    # manager to choose its own port.
    if object_manager_port is None:
        object_manager_port = 0

    if min_worker_port is None:
        min_worker_port = 0

    if max_worker_port is None:
        max_worker_port = 0

    if load_code_from_local:
        start_worker_command += ["--load-code-from-local"]

    command = [
        RAYLET_EXECUTABLE,
        "--raylet_socket_name={}".format(raylet_name),
        "--store_socket_name={}".format(plasma_store_name),
        "--object_manager_port={}".format(object_manager_port),
        "--min_worker_port={}".format(min_worker_port),
        "--max_worker_port={}".format(max_worker_port),
        "--node_manager_port={}".format(node_manager_port),
        "--node_ip_address={}".format(node_ip_address),
        "--redis_address={}".format(gcs_ip_address),
        "--redis_port={}".format(gcs_port),
        "--num_initial_workers={}".format(num_initial_workers),
        "--maximum_startup_concurrency={}".format(maximum_startup_concurrency),
        "--static_resource_list={}".format(resource_argument),
        "--config_list={}".format(config_str),
        "--python_worker_command={}".format(
            subprocess.list2cmdline(start_worker_command)
        ),
        "--java_worker_command={}".format(subprocess.list2cmdline(java_worker_command)),
        "--redis_password={}".format(redis_password or ""),
        "--temp_dir={}".format(temp_dir),
        "--session_dir={}".format(session_dir),
    ]
    process_info = start_ray_process(
        command,
        ray_constants.PROCESS_TYPE_RAYLET,
        use_valgrind=use_valgrind,
        use_gdb=False,
        use_valgrind_profiler=use_profiler,
        use_perftools_profiler=("RAYLET_PERFTOOLS_PATH" in os.environ),
        stdout_file=stdout_file,
        stderr_file=stderr_file,
        fate_share=fate_share,
    )

    return process_info


# Traceback (TimeoutError)
�[2m�[33m(pid=raylet)�[0m E0429 02:32:06.263886 22036 process.cc:274] Failed to wait for process 22047 with error system:10: No child processes
E0429 02:32:12.346844 23272 task_manager.cc:288] 3 retries left for task b48f33dc1265b526ffffffff0100, attempting to resubmit.
E0429 02:32:12.346899 23272 core_worker.cc:373] Will resubmit task after a 5000ms delay: Type=NORMAL_TASK, Language=PYTHON, function_descriptor={type=PythonFunctionDescriptor, module_name=__main__, class_name=, function_name=f, function_hash=7d2c6c88e5e801d48a350076f2117e717fe12224}, task_id=b48f33dc1265b526ffffffff0100, job_id=0100, num_args=2, num_returns=1
�[2m�[33m(pid=raylet)�[0m E0429 02:32:12.347446 22089 process.cc:274] Failed to wait for process 22100 with error system:10: No child processes
2020-04-29 02:32:12,653	INFO resource_spec.py:212 -- Starting Ray with 27.88 GiB memory available for workers and up to 0.15 GiB for objects. You can adjust these settings with ray.init(memory=<bytes>, object_store_memory=<bytes>).
�[2m�[33m(pid=raylet)�[0m E0429 02:32:12.732946757   22142 server_chttp2.cc:40]        {"created":"@1588127532.732848116","description":"No address added out of total 1 resolved","file":"external/com_github_grpc_grpc/src/core/ext/transport/chttp2/server/chttp2_server.cc","file_line":394,"referenced_errors":[{"created":"@1588127532.732846227","description":"Failed to add any wildcard listeners","file":"external/com_github_grpc_grpc/src/core/lib/iomgr/tcp_server_posix.cc","file_line":341,"referenced_errors":[{"created":"@1588127532.732832876","description":"Unable to configure socket","fd":44,"file":"external/com_github_grpc_grpc/src/core/lib/iomgr/tcp_server_utils_posix_common.cc","file_line":208,"referenced_errors":[{"created":"@1588127532.732823689","description":"Address already in use","errno":98,"file":"external/com_github_grpc_grpc/src/core/lib/iomgr/tcp_server_utils_posix_common.cc","file_line":181,"os_error":"Address already in use","syscall":"bind"}]},{"created":"@1588127532.732845812","description":"Unable to configure socket","fd":44,"file":"external/com_github_grpc_grpc/src/core/lib/iomgr/tcp_server_utils_posix_common.cc","file_line":208,"referenced_errors":[{"created":"@1588127532.732843382","description":"Address already in use","errno":98,"file":"external/com_github_grpc_grpc/src/core/lib/iomgr/tcp_server_utils_posix_common.cc","file_line":181,"os_error":"Address already in use","syscall":"bind"}]}]}]}
�[2m�[33m(pid=raylet)�[0m *** Aborted at 1588127532 (unix time) try "date -d @1588127532" if you are using GNU date ***
�[2m�[33m(pid=raylet)�[0m PC: @                0x0 (unknown)
�[2m�[33m(pid=raylet)�[0m *** SIGSEGV (@0x58) received by PID 22142 (TID 0x7fc3a66d37c0) from PID 88; stack trace: ***
�[2m�[33m(pid=raylet)�[0m     @     0x7fc3a5c32390 (unknown)
�[2m�[33m(pid=raylet)�[0m     @     0x5596e3957692 grpc::ServerInterface::RegisteredAsyncRequest::IssueRequest()
�[2m�[33m(pid=raylet)�[0m     @     0x5596e35b2149 ray::rpc::NodeManagerService::WithAsyncMethod_RequestWorkerLease<>::RequestRequestWorkerLease()
�[2m�[33m(pid=raylet)�[0m     @     0x5596e35c7b1b ray::rpc::ServerCallFactoryImpl<>::CreateCall()
�[2m�[33m(pid=raylet)�[0m     @     0x5596e380bfe1 ray::rpc::GrpcServer::Run()
�[2m�[33m(pid=raylet)�[0m     @     0x5596e3629acc ray::raylet::NodeManager::NodeManager()
�[2m�[33m(pid=raylet)�[0m     @     0x5596e35cbc07 ray::raylet::Raylet::Raylet()
�[2m�[33m(pid=raylet)�[0m     @     0x5596e359848d main
�[2m�[33m(pid=raylet)�[0m     @     0x7fc3a5459830 __libc_start_main
�[2m�[33m(pid=raylet)�[0m     @     0x5596e35a9391 (unknown)
�[2m�[36m(pid=22153)�[0m E0429 02:32:13.864451 22153 raylet_client.cc:69] Retrying to connect to socket for pathname /tmp/ray/session_2020-04-28_20-19-44_770473_22870/sockets/raylet.8 (num_attempts = 1, num_retries = 5)
�[2m�[36m(pid=22153)�[0m E0429 02:32:14.364712 22153 raylet_client.cc:69] Retrying to connect to socket for pathname /tmp/ray/session_2020-04-28_20-19-44_770473_22870/sockets/raylet.8 (num_attempts = 2, num_retries = 5)
�[2m�[36m(pid=22153)�[0m E0429 02:32:14.864863 22153 raylet_client.cc:69] Retrying to connect to socket for pathname /tmp/ray/session_2020-04-28_20-19-44_770473_22870/sockets/raylet.8 (num_attempts = 3, num_retries = 5)
�[2m�[36m(pid=22153)�[0m E0429 02:32:15.365000 22153 raylet_client.cc:69] Retrying to connect to socket for pathname /tmp/ray/session_2020-04-28_20-19-44_770473_22870/sockets/raylet.8 (num_attempts = 4, num_retries = 5)
�[2m�[36m(pid=22153)�[0m F0429 02:32:15.865115 22153 raylet_client.cc:78] Could not connect to socket /tmp/ray/session_2020-04-28_20-19-44_770473_22870/sockets/raylet.8
�[2m�[36m(pid=22153)�[0m *** Check failure stack trace: ***
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3b2b40ed  google::LogMessage::Fail()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3b2b555c  google::LogMessage::SendToLog()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3b2b3dc9  google::LogMessage::Flush()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3b2b3fe1  google::LogMessage::~LogMessage()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3b03bb39  ray::RayLog::~RayLog()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3ae55133  ray::raylet::RayletConnection::RayletConnection()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3ae55abf  ray::raylet::RayletClient::RayletClient()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3adf513b  ray::CoreWorker::CoreWorker()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3adf8984  ray::CoreWorkerProcess::CreateWorker()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3adf8efb  ray::CoreWorkerProcess::CoreWorkerProcess()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3adf93fb  ray::CoreWorkerProcess::Initialize()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3ad6c06c  __pyx_pw_3ray_7_raylet_10CoreWorker_1__cinit__()
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3ad6d155  __pyx_tp_new_3ray_7_raylet_CoreWorker()
�[2m�[36m(pid=22153)�[0m     @     0x55db24e47965  type_call
�[2m�[36m(pid=22153)�[0m     @     0x55db24db7d7b  _PyObject_FastCallDict
�[2m�[36m(pid=22153)�[0m     @     0x55db24e477ce  call_function
�[2m�[36m(pid=22153)�[0m     @     0x55db24e69cba  _PyEval_EvalFrameDefault
�[2m�[36m(pid=22153)�[0m     @     0x55db24e40dae  _PyEval_EvalCodeWithName
�[2m�[36m(pid=22153)�[0m     @     0x55db24e41941  fast_function
�[2m�[36m(pid=22153)�[0m     @     0x55db24e47755  call_function
�[2m�[36m(pid=22153)�[0m     @     0x55db24e6aa7a  _PyEval_EvalFrameDefault
�[2m�[36m(pid=22153)�[0m     @     0x55db24e42459  PyEval_EvalCodeEx
�[2m�[36m(pid=22153)�[0m     @     0x55db24e431ec  PyEval_EvalCode
�[2m�[36m(pid=22153)�[0m     @     0x55db24ebd9a4  run_mod
�[2m�[36m(pid=22153)�[0m     @     0x55db24ebdda1  PyRun_FileExFlags
�[2m�[36m(pid=22153)�[0m     @     0x55db24ebdfa4  PyRun_SimpleFileExFlags
�[2m�[36m(pid=22153)�[0m     @     0x55db24ec1a9e  Py_Main
�[2m�[36m(pid=22153)�[0m     @     0x55db24d894be  main
�[2m�[36m(pid=22153)�[0m     @     0x7f5d3cd85830  __libc_start_main
�[2m�[36m(pid=22153)�[0m     @     0x55db24e70773  (unknown)
Traceback (most recent call last):
File "workloads/node_failures.py", line 57, in <module>
cluster.add_node()
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ray/cluster_utils.py", line 115, in add_node
self._wait_for_node(node)
File "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ray/cluster_utils.py", line 165, in _wait_for_node
raise TimeoutError("Timed out while waiting for nodes to join.")
TimeoutError: Timed out while waiting for nodes to join.
�[2m�[33m(pid=raylet)�[0m E0429 02:32:42.965368 13125 process.cc:274] Failed to wait for process 13136 with error system:10: No child processes
�[2m�[33m(pid=raylet)�[0m E0429 02:32:43.045863  1167 process.cc:274] Failed to wait for process 1178 with error system:10: No child processes
2020-04-29 02:32:43,942	ERROR import_thread.py:93 -- ImportThread: Connection closed by server.
2020-04-29 02:32:43,942	ERROR worker.py:996 -- print_logs: Connection closed by server.
2020-04-29 02:32:43,942	ERROR worker.py:1096 -- listen_error_messages_raylet: Connection closed by server.
E0429 02:32:45.999132 22870 raylet_client.cc:90] IOError: [RayletClient] Connection closed unexpectedly. [RayletClient] Failed to disconnect from raylet.

# Fixed code
def start_raylet(
    redis_address,
    node_ip_address,
    node_manager_port,
    raylet_name,
    plasma_store_name,
    worker_path,
    temp_dir,
    session_dir,
    resource_spec,
    min_worker_port=None,
    max_worker_port=None,
    object_manager_port=None,
    redis_password=None,
    use_valgrind=False,
    use_profiler=False,
    stdout_file=None,
    stderr_file=None,
    config=None,
    include_java=False,
    java_worker_options=None,
    load_code_from_local=False,
    fate_share=None,
    socket_to_use=None,
):
    """Start a raylet, which is a combined local scheduler and object manager.

    Args:
        redis_address (str): The address of the primary Redis server.
        node_ip_address (str): The IP address of this node.
        node_manager_port(int): The port to use for the node manager. This must
            not be 0.
        raylet_name (str): The name of the raylet socket to create.
        plasma_store_name (str): The name of the plasma store socket to connect
             to.
        worker_path (str): The path of the Python file that new worker
            processes will execute.
        temp_dir (str): The path of the temporary directory Ray will use.
        session_dir (str): The path of this session.
        resource_spec (ResourceSpec): Resources for this raylet.
        object_manager_port: The port to use for the object manager. If this is
            None, then the object manager will choose its own port.
        min_worker_port (int): The lowest port number that workers will bind
            on. If not set, random ports will be chosen.
        max_worker_port (int): The highest port number that workers will bind
            on. If set, min_worker_port must also be set.
        redis_password: The password to use when connecting to Redis.
        use_valgrind (bool): True if the raylet should be started inside
            of valgrind. If this is True, use_profiler must be False.
        use_profiler (bool): True if the raylet should be started inside
            a profiler. If this is True, use_valgrind must be False.
        stdout_file: A file handle opened for writing to redirect stdout to. If
            no redirection should happen, then this should be None.
        stderr_file: A file handle opened for writing to redirect stderr to. If
            no redirection should happen, then this should be None.
        config (dict|None): Optional Raylet configuration that will
            override defaults in RayConfig.
        include_java (bool): If True, the raylet backend can also support
            Java worker.
        java_worker_options (list): The command options for Java worker.
    Returns:
        ProcessInfo for the process that was started.
    """
    # The caller must provide a node manager port so that we can correctly
    # populate the command to start a worker.
    assert node_manager_port is not None and node_manager_port != 0
    config = config or {}
    config_str = ",".join(["{},{}".format(*kv) for kv in config.items()])

    if use_valgrind and use_profiler:
        raise ValueError("Cannot use valgrind and profiler at the same time.")

    assert resource_spec.resolved()
    num_initial_workers = resource_spec.num_cpus
    static_resources = resource_spec.to_resource_dict()

    # Limit the number of workers that can be started in parallel by the
    # raylet. However, make sure it is at least 1.
    num_cpus_static = static_resources.get("CPU", 0)
    maximum_startup_concurrency = max(
        1, min(multiprocessing.cpu_count(), num_cpus_static)
    )

    # Format the resource argument in a form like 'CPU,1.0,GPU,0,Custom,3'.
    resource_argument = ",".join(
        ["{},{}".format(*kv) for kv in static_resources.items()]
    )

    gcs_ip_address, gcs_port = redis_address.split(":")

    if include_java is True:
        default_cp = os.pathsep.join(DEFAULT_JAVA_WORKER_CLASSPATH)
        java_worker_command = build_java_worker_command(
            json.loads(java_worker_options)
            if java_worker_options
            else ["-classpath", default_cp],
            redis_address,
            node_manager_port,
            plasma_store_name,
            raylet_name,
            redis_password,
            session_dir,
        )
    else:
        java_worker_command = []

    # Create the command that the Raylet will use to start workers.
    start_worker_command = [
        sys.executable,
        worker_path,
        "--node-ip-address={}".format(node_ip_address),
        "--node-manager-port={}".format(node_manager_port),
        "--object-store-name={}".format(plasma_store_name),
        "--raylet-name={}".format(raylet_name),
        "--redis-address={}".format(redis_address),
        "--config-list={}".format(config_str),
        "--temp-dir={}".format(temp_dir),
    ]
    if redis_password:
        start_worker_command += ["--redis-password={}".format(redis_password)]

    # If the object manager port is None, then use 0 to cause the object
    # manager to choose its own port.
    if object_manager_port is None:
        object_manager_port = 0

    if min_worker_port is None:
        min_worker_port = 0

    if max_worker_port is None:
        max_worker_port = 0

    if load_code_from_local:
        start_worker_command += ["--load-code-from-local"]

    command = [
        RAYLET_EXECUTABLE,
        "--raylet_socket_name={}".format(raylet_name),
        "--store_socket_name={}".format(plasma_store_name),
        "--object_manager_port={}".format(object_manager_port),
        "--min_worker_port={}".format(min_worker_port),
        "--max_worker_port={}".format(max_worker_port),
        "--node_manager_port={}".format(node_manager_port),
        "--node_ip_address={}".format(node_ip_address),
        "--redis_address={}".format(gcs_ip_address),
        "--redis_port={}".format(gcs_port),
        "--num_initial_workers={}".format(num_initial_workers),
        "--maximum_startup_concurrency={}".format(maximum_startup_concurrency),
        "--static_resource_list={}".format(resource_argument),
        "--config_list={}".format(config_str),
        "--python_worker_command={}".format(
            subprocess.list2cmdline(start_worker_command)
        ),
        "--java_worker_command={}".format(subprocess.list2cmdline(java_worker_command)),
        "--redis_password={}".format(redis_password or ""),
        "--temp_dir={}".format(temp_dir),
        "--session_dir={}".format(session_dir),
    ]
    if socket_to_use:
        socket_to_use.close()
    process_info = start_ray_process(
        command,
        ray_constants.PROCESS_TYPE_RAYLET,
        use_valgrind=use_valgrind,
        use_gdb=False,
        use_valgrind_profiler=use_profiler,
        use_perftools_profiler=("RAYLET_PERFTOOLS_PATH" in os.environ),
        stdout_file=stdout_file,
        stderr_file=stderr_file,
        fate_share=fate_share,
    )

    return process_info