Repository: googleapis/google-cloud-python

Issue:
Uncaught exceptions within the streaming pull code.
This comes from a [StackOverflow question](https://stackoverflow.com/questions/55552606/how-can-one-catch-exceptions-in-python-pubsub-subscriber-that-are-happening-in-i). There are internal exceptions that are not being caught and result in the client library no longer delivery messages.

```
Exception in thread Thread-LeaseMaintainer:
Traceback (most recent call last):
  File "/usr/local/lib/python3.6/site-packages/google/api_core/grpc_helpers.py", line 57, in error_remapped_callable
    return callable_(*args, **kwargs)
  File "/usr/local/lib/python3.6/site-packages/grpc/_channel.py", line 549, in __call__
    return _end_unary_response_blocking(state, call, False, None)
  File "/usr/local/lib/python3.6/site-packages/grpc/_channel.py", line 466, in _end_unary_response_blocking
    raise _Rendezvous(state, None, None, deadline)
grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with:
    status = StatusCode.UNAVAILABLE
    details = "channel is in state TRANSIENT_FAILURE"
    debug_error_string = "{"created":"@1554568036.075280756","description":"channel is in state TRANSIENT_FAILURE","file":"src/core/ext/filters/client_channel/client_channel.cc","file_line":2294,"grpc_status":14}"
>

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/local/lib/python3.6/site-packages/google/api_core/retry.py", line 179, in retry_target
    return target()
  File "/usr/local/lib/python3.6/site-packages/google/api_core/timeout.py", line 214, in func_with_timeout
    return func(*args, **kwargs)
  File "/usr/local/lib/python3.6/site-packages/google/api_core/grpc_helpers.py", line 59, in error_remapped_callable
    six.raise_from(exceptions.from_grpc_error(exc), exc)
  File "<string>", line 3, in raise_from
google.api_core.exceptions.ServiceUnavailable: 503 channel is in state TRANSIENT_FAILURE

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/local/lib/python3.6/threading.py", line 916, in _bootstrap_inner
    self.run()
  File "/usr/local/lib/python3.6/threading.py", line 864, in run
    self._target(*self._args, **self._kwargs)
  File "/usr/local/lib/python3.6/site-packages/google/cloud/pubsub_v1/subscriber/_protocol/leaser.py", line 146, in maintain_leases
    [requests.ModAckRequest(ack_id, p99) for ack_id in ack_ids]
  File "/usr/local/lib/python3.6/site-packages/google/cloud/pubsub_v1/subscriber/_protocol/dispatcher.py", line 152, in modify_ack_deadline
    self._manager.send(request)
  File "/usr/local/lib/python3.6/site-packages/google/cloud/pubsub_v1/subscriber/_protocol/streaming_pull_manager.py", line 268, in send
    self._send_unary_request(request)
  File "/usr/local/lib/python3.6/site-packages/google/cloud/pubsub_v1/subscriber/_protocol/streaming_pull_manager.py", line 259, in _send_unary_request
    ack_deadline_seconds=deadline,
  File "/usr/local/lib/python3.6/site-packages/google/cloud/pubsub_v1/_gapic.py", line 45, in <lambda>
    fx = lambda self, *a, **kw: wrapped_fx(self.api, *a, **kw)  # noqa
  File "/usr/local/lib/python3.6/site-packages/google/cloud/pubsub_v1/gapic/subscriber_client.py", line 723, in modify_ack_deadline
    request, retry=retry, timeout=timeout, metadata=metadata
  File "/usr/local/lib/python3.6/site-packages/google/api_core/gapic_v1/method.py", line 143, in __call__
    return wrapped_func(*args, **kwargs)
  File "/usr/local/lib/python3.6/site-packages/google/api_core/retry.py", line 270, in retry_wrapped_func
    on_error=on_error,
  File "/usr/local/lib/python3.6/site-packages/google/api_core/retry.py", line 199, in retry_target
    last_exc,
  File "<string>", line 3, in raise_from
google.api_core.exceptions.RetryError: Deadline of 600.0s exceeded while calling functools.partial(<function _wrap_unary_errors.<locals>.error_remapped_callable at 0x7f86228cd400>, subscription: "projects/xxxxx-dev/subscriptions/telemetry-sub"
ack_deadline_seconds: 10
ack_ids: "QBJMJwFESVMrQwsqWBFOBCEhPjA-RVNEUAYWLF1GSFE3GQhoUQ5PXiM_NSAoRRoHIGoKOUJdEmJoXFx1B1ALEHQoYnxvWRYFCEdReF1YHQdodGxXOFUEHnN1Y3xtWhQDAEFXf3f8gIrJ38BtZho9WxJLLD5-LDRFQV4"
, metadata=[('x-goog-api-client', 'gl-python/3.6.8 grpc/1.19.0 gax/1.8.2 gapic/0.40.0')]), last exception: 503 channel is in state TRANSIENT_FAILURE

Thread-ConsumeBidirectionalStream caught unexpected exception Deadline of 600.0s exceeded while calling functools.partial(<function _wrap_unary_errors.<locals>.error_remapped_callable at 0x7f86228cda60>, subscription: "projects/xxxxx-dev/subscriptions/telemetry-sub"
ack_deadline_seconds: 10
ack_ids: "QBJMJwFESVMrQwsqWBFOBCEhPjA-RVNEUAYWLF1GSFE3GQhoUQ5PXiM_NSAoRRoHIGoKOUJdEmJoXFx1B1ALEHQoYnxvWRYFCEdReF1YHAdodGxXOFUEHnN1aXVoWxAIBEdXeXf8gIrJ38BtZho9WxJLLD5-LDRFQV4"
, metadata=[('x-goog-api-client', 'gl-python/3.6.8 grpc/1.19.0 gax/1.8.2 gapic/0.40.0')]), last exception: 503 channel is in state TRANSIENT_FAILURE and will exit.
```

The user who reported the error was using the following versions:

python == 3.6.5
google-cloud-pubsub == 0.40.0 # but this has behaved similarly for at least the last several versions
google-api-core == 1.8.2
google-api-python-client == 1.7.8


Discussion:
We're seeing this behavior with the following stack trace as well. Errors are thrown and the subscriber stops receiving messages without the main thread (future.result(timeout=x)) raising an exception (other than the timeout). 

``` 
Error in queue callback worker: Deadline of 600.0s exceeded while calling functools.partial(<function _wrap_unary_errors.<locals>.error_remapped_callable at 0x7f6441ece6a8>, subscription: "projects/my-project/subscriptions/my-subscription"
ack_ids: "IkVBXkASTCQXRElTK0MLKlgRTgQhIT4wPkVTRFAGFixdRkhRNxkIaFEOT14jPzUgKEUSAABneSQZRhIKB1xcdQdRDB8jfjV2a1NBUgRPU3RfcysvV1ledwNWDx17fWd2a18TCSq8gaXa0elrZh49WhJLLD5-Lw"
, metadata=[('x-goog-api-client', 'gl-python/3.6.8 grpc/1.19.0 gax/1.8.0 gapic/0.39.1')]), last exception: 503 channel is in state TRANSIENT_FAILURE
Traceback (most recent call last):
  File "/usr/local/lib/python3.6/site-packages/google/api_core/grpc_helpers.py", line 57, in error_remapped_callable
    return callable_(*args, **kwargs)
  File "/usr/local/lib/python3.6/site-packages/grpc/_channel.py", line 549, in __call__
    return _end_unary_response_blocking(state, call, False, None)
  File "/usr/local/lib/python3.6/site-packages/grpc/_channel.py", line 466, in _end_unary_response_blocking
    raise _Rendezvous(state, None, None, deadline)
grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with:
	status = StatusCode.UNAVAILABLE
	details = "channel is in state TRANSIENT_FAILURE"
	debug_error_string = "{"created":"@1555805428.654693392","description":"channel is in state TRANSIENT_FAILURE","file":"src/core/ext/filters/client_channel/client_channel.cc","file_line":2294,"grpc_status":14}"
>

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/local/lib/python3.6/site-packages/google/api_core/retry.py", line 179, in retry_target
    return target()
  File "/usr/local/lib/python3.6/site-packages/google/api_core/timeout.py", line 214, in func_with_timeout
    return func(*args, **kwargs)
  File "/usr/local/lib/python3.6/site-packages/google/api_core/grpc_helpers.py", line 59, in error_remapped_callable
    six.raise_from(exceptions.from_grpc_error(exc), exc)
  File "<string>", line 3, in raise_from
google.api_core.exceptions.ServiceUnavailable: 503 channel is in state TRANSIENT_FAILURE

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/local/lib/python3.6/site-packages/google/cloud/pubsub_v1/subscriber/_protocol/helper_threads.py", line 108, in __call__
    self._callback(items)
  File "/usr/local/lib/python3.6/site-packages/google/cloud/pubsub_v1/subscriber/_protocol/dispatcher.py", line 97, in dispatch_callback
    self.ack(batched_commands.pop(requests.AckRequest))
  File "/usr/local/lib/python3.6/site-packages/google/cloud/pubsub_v1/subscriber/_protocol/dispatcher.py", line 117, in ack
    self._manager.send(request)
  File "/usr/local/lib/python3.6/site-packages/google/cloud/pubsub_v1/subscriber/_protocol/streaming_pull_manager.py", line 267, in send
    self._send_unary_request(request)
  File "/usr/local/lib/python3.6/site-packages/google/cloud/pubsub_v1/subscriber/_protocol/streaming_pull_manager.py", line 243, in _send_unary_request
    subscription=self._subscription, ack_ids=list(request.ack_ids)
  File "/usr/local/lib/python3.6/site-packages/google/cloud/pubsub_v1/_gapic.py", line 45, in <lambda>
    fx = lambda self, *a, **kw: wrapped_fx(self.api, *a, **kw)  # noqa
  File "/usr/local/lib/python3.6/site-packages/google/cloud/pubsub_v1/gapic/subscriber_client.py", line 788, in acknowledge
    request, retry=retry, timeout=timeout, metadata=metadata
  File "/usr/local/lib/python3.6/site-packages/google/api_core/gapic_v1/method.py", line 143, in __call__
    return wrapped_func(*args, **kwargs)
  File "/usr/local/lib/python3.6/site-packages/google/api_core/retry.py", line 270, in retry_wrapped_func
    on_error=on_error,
  File "/usr/local/lib/python3.6/site-packages/google/api_core/retry.py", line 199, in retry_target
    last_exc,
  File "<string>", line 3, in raise_from
google.api_core.exceptions.RetryError: Deadline of 600.0s exceeded while calling functools.partial(<function _wrap_unary_errors.<locals>.error_remapped_callable at 0x7f6441ece6a8>, subscription: "projects/my-project/subscriptions/my-subsciption"
ack_ids: "IkVBXkASTCQXRElTK0MLKlgRTgQhIT4wPkVTRFAGFixdRkhRNxkIaFEOT14jPzUgKEUSAABneSQZRhIKB1xcdQdRDB8jfjV2a1NBUgRPU3RfcysvV1ledwNWDx17fWd2a18TCSq8gaXa0elrZh49WhJLLD5-Lw"
, metadata=[('x-goog-api-client', 'gl-python/3.6.8 grpc/1.19.0 gax/1.8.0 gapic/0.39.1')]), last exception: 503 channel is in state TRANSIENT_FAILURE
```

Here's what our code looks like:
```
future = client.subscribe(path, callback=callback, flow_control=flow_control)

while True:
        try:
            future.result(timeout=1)
        except pubsub_v1.exceptions.TimeoutError:
            pass
         except Exception as exc:
            logger.exception('Got uncaught exception in subscriber or callback'
                             f'; Traceback:{traceback.format_exc()} Retrying '
                             f'in 1s. Detail: {exc}')
            time.sleep(1)
            consecutive_errors += 1
            last_error = exc
            continue
        else:
            consecutive_errors = 0
        if consecutive_errors > MAX_CONSECUTIVE_ERRORS:
            logger.exception('Too many consecutive errors in PubSub consumer: '
                             f'{consecutive_errors}. Last exc: {last_error} '
                             'Re-raising last.')
            raise  #re-raise whatever last error was
        sleep(...)
```  
The Exception branch is never taken, and the callback ceases to be called after such an error, requiring main thread restart.

I believe this error occurs if the underlying channel enters the TRANSIENT_FAILURE state and remains in it for too long, i.e. longer than the `total_timeout_millis` setting of the subscriber client.

I was not able to produce the bug with a sample pub/sub application running on Kubernetes, but I did manage to trigger the reported scenario locally by doing the the following:
* Patch the [_end_unary_response_blocking()](https://github.com/grpc/grpc/blob/master/src/python/grpcio/grpc/_channel.py#L459-L467) function of the `grpc` dependency with the following:
```patch
--- /home/peter/workspace/google-cloud-python/venv-3.6/lib/python3.6/site-packages/grpc/_channel.py     2019-04-23 17:01:39.282064676 +0200
+++ /home/peter/workspace/google-cloud-python/venv-3.6/lib/python3.6/site-packages/grpc/_channel.py     2019-04-25 15:49:05.220317794 +0200
@@ -456,6 +456,16 @@
 
 
 def _end_unary_response_blocking(state, call, with_call, deadline):
+    #####################
+    import datetime
+    minute = datetime.datetime.now().minute
+    if 45 <= minute <= 56:
+        state.code = grpc.StatusCode.UNAVAILABLE
+        state.details = "channel is in **fake** TRANSIENT_FAILURE state"
+        state.debug_error_string = (
+            "transient failure is faked during a fixed time window in an hour"
+        )
+    ###########################
     if state.code is grpc.StatusCode.OK:
         if with_call:
             rendezvous = _Rendezvous(state, call, None, deadline)
```
The patch fakes a channel error during particular minutes in an hour (adjust as necessary).
* Start publishing messages and start a subscriber using the streaming pull (FWIW, I used my own test publisher and subscriber - [link](https://github.com/plamut/google-cloud-python/tree/bug-sub-hanging/pubsub/docker-app)).
* Wait for 10 minutes and some moderate amount of time more, then check the logs.

**Result:**
Eventually a `RetryError` is raised, and several threads exit (e.g. `Thread-ConsumeBidirectionalStream`). However, the main thread keeps running, but no messages are received and processed anymore. Subscriber must be manually be stopped with <kbd>Ctrl</kbd>+<kbd>C</kbd>.

> NOTE: It is not necessary to wait for full 10+ minutes, one can reduce the `total_timeout_millis` setting in [subscriber settings](https://github.com/googleapis/google-cloud-python/blob/master/pubsub/google/cloud/pubsub_v1/gapic/subscriber_client_config.py#L9-L36).

--- 

What happens is that if the subscriber has been retrying for too long, a [RetryError](https://github.com/googleapis/google-cloud-python/blob/master/api_core/google/api_core/exceptions.py#L45) is raised in the [retry wrapper](https://github.com/googleapis/google-cloud-python/blob/master/api_core/google/api_core/retry.py#L191-L200). This error is considered non-retryable, and the subscriber stopping pulling messages is actually expected behavior IMO. Will look into it.

What _should_ happen, however, is propagating the error to the main thread (and shutting everything down cleanly in the background), giving users a chance to catch the error and react to it as they see fit. 

Will discuss if this is the the expected way of handling this, and then work on a fix. Thank you for reporting the issue!
@plamut Thanks very much for digging into this!  A bit surprised my team are the first to report it. Please do post here if we can provide more details or testing, or when a fix or an ETA is available. :)
@jakeczyz AFAIK there have been several independent reports of the same (or similar) bug in the past, including the non-Python clients, but it was (very) difficult to reproduce it. I could not reproduce it either, thus only _suspect_ that this is the true cause kicking in on random occasions. The tracebacks are very similar, though, which is promising.

I do not have an ETA yet, but expect to discuss this with others next week - will post more, when I know more. :)

Resolving patch:
diff --git a/pubsub/google/cloud/pubsub_v1/subscriber/_protocol/streaming_pull_manager.py b/pubsub/google/cloud/pubsub_v1/subscriber/_protocol/streaming_pull_manager.py
--- a/pubsub/google/cloud/pubsub_v1/subscriber/_protocol/streaming_pull_manager.py
+++ b/pubsub/google/cloud/pubsub_v1/subscriber/_protocol/streaming_pull_manager.py
@@ -262,7 +262,11 @@ def _send_unary_request(self, request):
         _LOGGER.debug("Sent request(s) over unary RPC.")
 
     def send(self, request):
-        """Queue a request to be sent to the RPC."""
+        """Queue a request to be sent to the RPC.
+
+        If a RetryError occurs, the manager shutdown is triggered, and the
+        error is re-raised.
+        """
         if self._UNARY_REQUESTS:
             try:
                 self._send_unary_request(request)
@@ -272,6 +276,17 @@ def send(self, request):
                     "non-fatal as stream requests are best-effort.",
                     exc_info=True,
                 )
+            except exceptions.RetryError as exc:
+                _LOGGER.debug(
+                    "RetryError while sending unary RPC. Waiting on a transient "
+                    "error resolution for too long, will now trigger shutdown.",
+                    exc_info=False,
+                )
+                # The underlying channel has been suffering from a retryable error
+                # for too long, time to give up and shut the streaming pull down.
+                self._on_rpc_done(exc)
+                raise
+
         else:
             self._rpc.send(request)
 


Test patch:
diff --git a/pubsub/tests/unit/pubsub_v1/subscriber/test_streaming_pull_manager.py b/pubsub/tests/unit/pubsub_v1/subscriber/test_streaming_pull_manager.py
--- a/pubsub/tests/unit/pubsub_v1/subscriber/test_streaming_pull_manager.py
+++ b/pubsub/tests/unit/pubsub_v1/subscriber/test_streaming_pull_manager.py
@@ -245,7 +245,7 @@ def test_send_unary_empty():
     manager._client.modify_ack_deadline.assert_not_called()
 
 
-def test_send_unary_error(caplog):
+def test_send_unary_api_call_error(caplog):
     caplog.set_level(logging.DEBUG)
 
     manager = make_manager()
@@ -259,6 +259,24 @@ def test_send_unary_error(caplog):
     assert "The front fell off" in caplog.text
 
 
+def test_send_unary_retry_error(caplog):
+    caplog.set_level(logging.DEBUG)
+
+    manager, _, _, _, _, _ = make_running_manager()
+    manager._UNARY_REQUESTS = True
+
+    error = exceptions.RetryError(
+        "Too long a transient error", cause=Exception("Out of time!")
+    )
+    manager._client.acknowledge.side_effect = error
+
+    with pytest.raises(exceptions.RetryError):
+        manager.send(types.StreamingPullRequest(ack_ids=["ack_id1", "ack_id2"]))
+
+    assert "RetryError while sending unary RPC" in caplog.text
+    assert "signaled streaming pull manager shutdown" in caplog.text
+
+
 def test_send_streaming():
     manager = make_manager()
     manager._UNARY_REQUESTS = False