# Buggy code
def __init__(
    self,
    conn_str: str = "",
    secure: bool = False,
    metadata: List[Tuple[str, str]] = None,
    connection_retries: int = 3,
):
    """Initializes the worker side grpc client.

    Args:
        conn_str: The host:port connection string for the ray server.
        secure: whether to use SSL secure channel or not.
        metadata: additional metadata passed in the grpc request headers.
        connection_retries: Number of times to attempt to reconnect to the
          ray server if it doesn't respond immediately. Setting to 0 tries
          at least once.  For infinite retries, catch the ConnectionError
          exception.
    """
    self.metadata = metadata if metadata else []
    self.channel = None
    self._client_id = make_client_id()
    if secure:
        credentials = grpc.ssl_channel_credentials()
        self.channel = grpc.secure_channel(conn_str, credentials)
    else:
        self.channel = grpc.insecure_channel(conn_str)

    # Retry the connection until the channel responds to something
    # looking like a gRPC connection, though it may be a proxy.
    conn_attempts = 0
    timeout = INITIAL_TIMEOUT_SEC
    ray_ready = False
    while conn_attempts < max(connection_retries, 1):
        conn_attempts += 1
        try:
            # Let gRPC wait for us to see if the channel becomes ready.
            # If it throws, we couldn't connect.
            grpc.channel_ready_future(self.channel).result(timeout=timeout)
            # The HTTP2 channel is ready. Wrap the channel with the
            # RayletDriverStub, allowing for unary requests.
            self.server = ray_client_pb2_grpc.RayletDriverStub(self.channel)
            # Now the HTTP2 channel is ready, or proxied, but the
            # servicer may not be ready. Call is_initialized() and if
            # it throws, the servicer is not ready. On success, the
            # `ray_ready` result is checked.
            ray_ready = self.is_initialized()
            if ray_ready:
                # Ray is ready! Break out of the retry loop
                break
            # Ray is not ready yet, wait a timeout
            time.sleep(timeout)
        except grpc.FutureTimeoutError:
            logger.info(f"Couldn't connect channel in {timeout} seconds, retrying")
            # Note that channel_ready_future constitutes its own timeout,
            # which is why we do not sleep here.
        except grpc.RpcError as e:
            if e.code() == grpc.StatusCode.UNAVAILABLE:
                # UNAVAILABLE is gRPC's retryable error,
                # so we do that here.
                logger.info(f"Ray client server unavailable, retrying in {timeout}s...")
                logger.debug(f"Received when checking init: {e.details()}")
                # Ray is not ready yet, wait a timeout
                time.sleep(timeout)
            else:
                # Any other gRPC error gets a reraise
                raise e
        # Fallthrough, backoff, and retry at the top of the loop
        logger.info(
            f"Waiting for Ray to become ready on the server, retry in {timeout}s..."
        )
        timeout = backoff(timeout)

    # If we made it through the loop without ray_ready it means we've used
    # up our retries and should error back to the user.
    if not ray_ready:
        raise ConnectionError("ray client connection timeout")

    # Initialize the streams to finish protocol negotiation.
    self.data_client = DataClient(self.channel, self._client_id, self.metadata)
    self.reference_count: Dict[bytes, int] = defaultdict(int)

    self.log_client = LogstreamClient(self.channel, self.metadata)
    self.log_client.set_logstream_level(logging.INFO)
    self.closed = False


# Traceback (debug_error)
Got Error from data channel -- shutting down: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNAVAILABLE
details = "Received http2 header with status: 502"
debug_error_string = "{"created":"@1610421939.184399675","description":"Received http2 :status header with non-200 OK status","file":"src/core/ext/filters/http/client/http_client_filter.cc","file_line":129,"grpc_message":"Received http2 header with status: 502","grpc_status":14,"value":"502"}"

Got Error from logger channel -- shutting down: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNAVAILABLE
details = "Received http2 header with status: 502"
debug_error_string = "{"created":"@1610421939.184408826","description":"Received http2 :status header with non-200 OK status","file":"src/core/ext/filters/http/client/http_client_filter.cc","file_line":129,"grpc_message":"Received http2 header with status: 502","grpc_status":14,"value":"502"}"

Exception in thread Thread-1:
Traceback (most recent call last):
File "/usr/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/usr/lib/python3.6/threading.py", line 864, in run
self._target(*self._args, **self._kwargs)
File "/home/eric/Desktop/ray/python/ray/util/client/dataclient.py", line 76, in _data_main
raise e
File "/home/eric/Desktop/ray/python/ray/util/client/dataclient.py", line 61, in _data_main
for response in resp_stream:
File "/home/eric/.local/lib/python3.6/site-packages/grpc/_channel.py", line 416, in __next__
return self._next()
File "/home/eric/.local/lib/python3.6/site-packages/grpc/_channel.py", line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNAVAILABLE
details = "Received http2 header with status: 502"
debug_error_string = "{"created":"@1610421939.184399675","description":"Received http2 :status header with non-200 OK status","file":"src/core/ext/filters/http/client/http_client_filter.cc","file_line":129,"grpc_message":"Received http2 header with status: 502","grpc_status":14,"value":"502"}"


Exception in thread Thread-2:
Traceback (most recent call last):
File "/usr/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/usr/lib/python3.6/threading.py", line 864, in run
self._target(*self._args, **self._kwargs)
File "/home/eric/Desktop/ray/python/ray/util/client/logsclient.py", line 51, in _log_main
raise e
File "/home/eric/Desktop/ray/python/ray/util/client/logsclient.py", line 42, in _log_main
for record in log_stream:
File "/home/eric/.local/lib/python3.6/site-packages/grpc/_channel.py", line 416, in __next__
return self._next()
File "/home/eric/.local/lib/python3.6/site-packages/grpc/_channel.py", line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNAVAILABLE
details = "Received http2 header with status: 502"
debug_error_string = "{"created":"@1610421939.184408826","description":"Received http2 :status header with non-200 OK status","file":"src/core/ext/filters/http/client/http_client_filter.cc","file_line":129,"grpc_message":"Received http2 header with status: 502","grpc_status":14,"value":"502"}"


# Fixed code
def __init__(
    self,
    conn_str: str = "",
    secure: bool = False,
    metadata: List[Tuple[str, str]] = None,
    connection_retries: int = 3,
):
    """Initializes the worker side grpc client.

    Args:
        conn_str: The host:port connection string for the ray server.
        secure: whether to use SSL secure channel or not.
        metadata: additional metadata passed in the grpc request headers.
        connection_retries: Number of times to attempt to reconnect to the
          ray server if it doesn't respond immediately. Setting to 0 tries
          at least once.  For infinite retries, catch the ConnectionError
          exception.
    """
    self.metadata = metadata if metadata else []
    self.channel = None
    self._conn_state = grpc.ChannelConnectivity.IDLE
    self._client_id = make_client_id()
    if secure:
        credentials = grpc.ssl_channel_credentials()
        self.channel = grpc.secure_channel(conn_str, credentials)
    else:
        self.channel = grpc.insecure_channel(conn_str)

    self.channel.subscribe(self._on_channel_state_change)

    # Retry the connection until the channel responds to something
    # looking like a gRPC connection, though it may be a proxy.
    conn_attempts = 0
    timeout = INITIAL_TIMEOUT_SEC
    ray_ready = False
    while conn_attempts < max(connection_retries, 1):
        conn_attempts += 1
        try:
            # Let gRPC wait for us to see if the channel becomes ready.
            # If it throws, we couldn't connect.
            grpc.channel_ready_future(self.channel).result(timeout=timeout)
            # The HTTP2 channel is ready. Wrap the channel with the
            # RayletDriverStub, allowing for unary requests.
            self.server = ray_client_pb2_grpc.RayletDriverStub(self.channel)
            # Now the HTTP2 channel is ready, or proxied, but the
            # servicer may not be ready. Call is_initialized() and if
            # it throws, the servicer is not ready. On success, the
            # `ray_ready` result is checked.
            ray_ready = self.is_initialized()
            if ray_ready:
                # Ray is ready! Break out of the retry loop
                break
            # Ray is not ready yet, wait a timeout
            time.sleep(timeout)
        except grpc.FutureTimeoutError:
            logger.info(f"Couldn't connect channel in {timeout} seconds, retrying")
            # Note that channel_ready_future constitutes its own timeout,
            # which is why we do not sleep here.
        except grpc.RpcError as e:
            if e.code() == grpc.StatusCode.UNAVAILABLE:
                # UNAVAILABLE is gRPC's retryable error,
                # so we do that here.
                logger.info(f"Ray client server unavailable, retrying in {timeout}s...")
                logger.debug(f"Received when checking init: {e.details()}")
                # Ray is not ready yet, wait a timeout
                time.sleep(timeout)
            else:
                # Any other gRPC error gets a reraise
                raise e
        # Fallthrough, backoff, and retry at the top of the loop
        logger.info(
            f"Waiting for Ray to become ready on the server, retry in {timeout}s..."
        )
        timeout = backoff(timeout)

    # If we made it through the loop without ray_ready it means we've used
    # up our retries and should error back to the user.
    if not ray_ready:
        raise ConnectionError("ray client connection timeout")

    # Initialize the streams to finish protocol negotiation.
    self.data_client = DataClient(self.channel, self._client_id, self.metadata)
    self.reference_count: Dict[bytes, int] = defaultdict(int)

    self.log_client = LogstreamClient(self.channel, self.metadata)
    self.log_client.set_logstream_level(logging.INFO)
    self.closed = False