# Buggy code
def _data_main(self) -> None:
    stub = ray_client_pb2_grpc.RayletDataStreamerStub(self.channel)
    resp_stream = stub.Datapath(
        iter(self.request_queue.get, None),
        metadata=[("client_id", self._client_id)] + self._metadata,
        wait_for_ready=True,
    )
    try:
        for response in resp_stream:
            if response.req_id == 0:
                # This is not being waited for.
                logger.debug(f"Got unawaited response {response}")
                continue
            with self.cv:
                self.ready_data[response.req_id] = response
                self.cv.notify_all()
    except grpc.RpcError as e:
        if grpc.StatusCode.CANCELLED == e.code():
            # Gracefully shutting down
            logger.info("Cancelling data channel")
        else:
            logger.error(f"Got Error from data channel -- shutting down: {e}")
            raise e


# Traceback (debug_error)
Got Error from data channel -- shutting down: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNAVAILABLE
details = "Received http2 header with status: 502"
debug_error_string = "{"created":"@1610421939.184399675","description":"Received http2 :status header with non-200 OK status","file":"src/core/ext/filters/http/client/http_client_filter.cc","file_line":129,"grpc_message":"Received http2 header with status: 502","grpc_status":14,"value":"502"}"

Got Error from logger channel -- shutting down: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNAVAILABLE
details = "Received http2 header with status: 502"
debug_error_string = "{"created":"@1610421939.184408826","description":"Received http2 :status header with non-200 OK status","file":"src/core/ext/filters/http/client/http_client_filter.cc","file_line":129,"grpc_message":"Received http2 header with status: 502","grpc_status":14,"value":"502"}"

Exception in thread Thread-1:
Traceback (most recent call last):
File "/usr/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/usr/lib/python3.6/threading.py", line 864, in run
self._target(*self._args, **self._kwargs)
File "/home/eric/Desktop/ray/python/ray/util/client/dataclient.py", line 76, in _data_main
raise e
File "/home/eric/Desktop/ray/python/ray/util/client/dataclient.py", line 61, in _data_main
for response in resp_stream:
File "/home/eric/.local/lib/python3.6/site-packages/grpc/_channel.py", line 416, in __next__
return self._next()
File "/home/eric/.local/lib/python3.6/site-packages/grpc/_channel.py", line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNAVAILABLE
details = "Received http2 header with status: 502"
debug_error_string = "{"created":"@1610421939.184399675","description":"Received http2 :status header with non-200 OK status","file":"src/core/ext/filters/http/client/http_client_filter.cc","file_line":129,"grpc_message":"Received http2 header with status: 502","grpc_status":14,"value":"502"}"


Exception in thread Thread-2:
Traceback (most recent call last):
File "/usr/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/usr/lib/python3.6/threading.py", line 864, in run
self._target(*self._args, **self._kwargs)
File "/home/eric/Desktop/ray/python/ray/util/client/logsclient.py", line 51, in _log_main
raise e
File "/home/eric/Desktop/ray/python/ray/util/client/logsclient.py", line 42, in _log_main
for record in log_stream:
File "/home/eric/.local/lib/python3.6/site-packages/grpc/_channel.py", line 416, in __next__
return self._next()
File "/home/eric/.local/lib/python3.6/site-packages/grpc/_channel.py", line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNAVAILABLE
details = "Received http2 header with status: 502"
debug_error_string = "{"created":"@1610421939.184408826","description":"Received http2 :status header with non-200 OK status","file":"src/core/ext/filters/http/client/http_client_filter.cc","file_line":129,"grpc_message":"Received http2 header with status: 502","grpc_status":14,"value":"502"}"


# Fixed code
def _data_main(self) -> None:
    stub = ray_client_pb2_grpc.RayletDataStreamerStub(self.channel)
    resp_stream = stub.Datapath(
        iter(self.request_queue.get, None),
        metadata=[("client_id", self._client_id)] + self._metadata,
        wait_for_ready=True,
    )
    try:
        for response in resp_stream:
            if response.req_id == 0:
                # This is not being waited for.
                logger.debug(f"Got unawaited response {response}")
                continue
            with self.cv:
                self.ready_data[response.req_id] = response
                self.cv.notify_all()
    except grpc.RpcError as e:
        with self.cv:
            self._in_shutdown = True
            self.cv.notify_all()
        if e.code() == grpc.StatusCode.CANCELLED:
            # Gracefully shutting down
            logger.info("Cancelling data channel")
        elif e.code() == grpc.StatusCode.UNAVAILABLE:
            # TODO(barakmich): The server may have
            # dropped. In theory, we can retry, as per
            # https://grpc.github.io/grpc/core/md_doc_statuscodes.html but
            # in practice we may need to think about the correct semantics
            # here.
            logger.info("Server disconnected from data channel")
        else:
            logger.error(f"Got Error from data channel -- shutting down: {e}")
            raise e