Repository: googleapis/python-storage

Issue:
Retry did not occur in case of google.auth.exceptions.TransportError
I am finding below error in one of my production environment using Cloud Functions. The sdk should retry by default but its not retrying.
I am trying to access a bucket in Cloud function python38 environment

Code Snippet
```python
googleCredentials = service_account.Credentials.from_service_account_info(authentication)
gcs = storage.Client(project=project, credentials=googleCredentials)
stage_bucket = gcs.get_bucket('test-bucket') 
```

My  package version
```
google-auth==1.24.0
google-cloud-storage==1.36.2
google-api-python-client==1.12.8
```

Error:
```
Error in app: Exception Traceback (most recent call last): File "/layers/google.python.pip/pip/lib/python3.8/site-packages/urllib3/connectionpool.py", line 670, in urlopen httplib_response = self._make_request( File "/layers/google.python.pip/pip/lib/python3.8/site-packages/urllib3/connectionpool.py", line 426, in _make_request six.raise_from(e, None) File "<string>", line 3, in raise_from File "/layers/google.python.pip/pip/lib/python3.8/site-packages/urllib3/connectionpool.py", line 421, in _make_request httplib_response = conn.getresponse() File "/opt/python3.8/lib/python3.8/http/client.py", line 1347, in getresponse response.begin() File "/opt/python3.8/lib/python3.8/http/client.py", line 307, in begin version, status, reason = self._read_status() File "/opt/python3.8/lib/python3.8/http/client.py", line 276, in _read_status raise RemoteDisconnected("Remote end closed connection without" http.client.RemoteDisconnected: Remote end closed connection without response During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/layers/google.python.pip/pip/lib/python3.8/site-packages/requests/adapters.py", line 439, in send resp = conn.urlopen( File "/layers/google.python.pip/pip/lib/python3.8/site-packages/urllib3/connectionpool.py", line 726, in urlopen retries = retries.increment( File "/layers/google.python.pip/pip/lib/python3.8/site-packages/urllib3/util/retry.py", line 410, in increment raise six.reraise(type(error), error, _stacktrace) File "/layers/google.python.pip/pip/lib/python3.8/site-packages/urllib3/packages/six.py", line 734, in reraise raise value.with_traceback(tb) File "/layers/google.python.pip/pip/lib/python3.8/site-packages/urllib3/connectionpool.py", line 670, in urlopen httplib_response = self._make_request( File "/layers/google.python.pip/pip/lib/python3.8/site-packages/urllib3/connectionpool.py", line 426, in _make_request six.raise_from(e, None) File "<string>", line 3, in raise_from File "/layers/google.python.pip/pip/lib/python3.8/site-packages/urllib3/connectionpool.py", line 421, in _make_request httplib_response = conn.getresponse() File "/opt/python3.8/lib/python3.8/http/client.py", line 1347, in getresponse response.begin() File "/opt/python3.8/lib/python3.8/http/client.py", line 307, in begin version, status, reason = self._read_status() File "/opt/python3.8/lib/python3.8/http/client.py", line 276, in _read_status raise RemoteDisconnected("Remote end closed connection without" urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response')) During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/auth/transport/requests.py", line 182, in __call__ response = self.session.request( File "/layers/google.python.pip/pip/lib/python3.8/site-packages/requests/sessions.py", line 530, in request resp = self.send(prep, **send_kwargs) File "/layers/google.python.pip/pip/lib/python3.8/site-packages/requests/sessions.py", line 643, in send r = adapter.send(request, **kwargs) File "/layers/google.python.pip/pip/lib/python3.8/site-packages/requests/adapters.py", line 498, in send raise ConnectionError(err, request=request) requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response')) The above exception was the direct cause of the following exception: Traceback (most recent call last): File "/workspace/common/functions.py", line 55, in wrapper response = wrapped_func(*args, **kwargs) File "/workspace/main.py", line 970, in manifest **stage_bucket = gcs.get_bucket("test-bucket")** File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/cloud/storage/client.py", line 402, in get_bucket bucket.reload( File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/cloud/storage/bucket.py", line 996, in reload super(Bucket, self).reload( File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/cloud/storage/_helpers.py", line 218, in reload api_response = client._connection.api_request( File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/cloud/storage/_http.py", line 78, in api_request return call() File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/api_core/retry.py", line 281, in retry_wrapped_func return retry_target( File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/api_core/retry.py", line 184, in retry_target return target() File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/cloud/_http.py", line 472, in api_request response = self._make_request( File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/cloud/_http.py", line 336, in _make_request return self._do_request( File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/cloud/_http.py", line 374, in _do_request return self.http.request( File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/auth/transport/requests.py", line 460, in request self.credentials.before_request(auth_request, method, url, request_headers) File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/auth/credentials.py", line 133, in before_request self.refresh(request) File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/oauth2/service_account.py", line 361, in refresh access_token, expiry, _ = _client.jwt_grant(request, self._token_uri, assertion) File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/oauth2/_client.py", line 153, in jwt_grant response_data = _token_endpoint_request(request, token_uri, body) File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/oauth2/_client.py", line 105, in _token_endpoint_request response = request(method="POST", url=token_uri, headers=headers, body=body) File "/layers/google.python.pip/pip/lib/python3.8/site-packages/google/auth/transport/requests.py", line 188, in __call__ six.raise_from(new_exc, caught_exc) File "<string>", line 3, in raise_from google.auth.exceptions.TransportError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
```



Discussion:
frankyn: @tswast I saw you had the same issue in bigquery. Could you take a look? I think this is a reasonable add given that this error only occurs in this case, but It might be better to restrict to specific types of connection errors. 

Patch:
diff --git a/google/cloud/storage/retry.py b/google/cloud/storage/retry.py
index e9a9eeb..e17f3d5 100644
--- a/google/cloud/storage/retry.py
+++ b/google/cloud/storage/retry.py
@@ -14,18 +14,19 @@
 
 import requests
 
-from google.api_core import exceptions
+from google.api_core import exceptions as api_exceptions
 from google.api_core import retry
+from google.auth import exceptions as auth_exceptions
 
 import json
 
 
 _RETRYABLE_TYPES = (
-    exceptions.TooManyRequests,  # 429
-    exceptions.InternalServerError,  # 500
-    exceptions.BadGateway,  # 502
-    exceptions.ServiceUnavailable,  # 503
-    exceptions.GatewayTimeout,  # 504
+    api_exceptions.TooManyRequests,  # 429
+    api_exceptions.InternalServerError,  # 500
+    api_exceptions.BadGateway,  # 502
+    api_exceptions.ServiceUnavailable,  # 503
+    api_exceptions.GatewayTimeout,  # 504
     requests.ConnectionError,
 )
 
@@ -37,8 +38,10 @@ def _should_retry(exc):
     """Predicate for determining when to retry."""
     if isinstance(exc, _RETRYABLE_TYPES):
         return True
-    elif isinstance(exc, exceptions.GoogleAPICallError):
+    elif isinstance(exc, api_exceptions.GoogleAPICallError):
         return exc.code in _ADDITIONAL_RETRYABLE_STATUS_CODES
+    elif isinstance(exc, auth_exceptions.TransportError):
+        return _should_retry(exc.args[0])
     else:
         return False
 


Test patch:
diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py
index d9a773c..582fa80 100644
--- a/tests/unit/test_retry.py
+++ b/tests/unit/test_retry.py
@@ -25,7 +25,16 @@ class Test_should_retry(unittest.TestCase):
 
         return retry._should_retry(exc)
 
-    def test_w_retryable_types(self):
+    def test_w_retryable_transport_error(self):
+        from google.cloud.storage import retry
+        from google.auth.exceptions import TransportError as eTransportError
+        from requests import ConnectionError as rConnectionError
+
+        caught_exc = rConnectionError("Remote end closed connection unexpected")
+        exc = eTransportError(caught_exc)
+        self.assertTrue(retry._should_retry(exc))
+
+    def test_w_wrapped_type(self):
         from google.cloud.storage import retry
 
         for exc_type in retry._RETRYABLE_TYPES: