Repository: googleapis/google-cloud-python

Issue:
Refactor apitools legacy 'handle_http_exceptions' for sanity.
The [apitools code which handles the various errors being raised when making a request](https://github.com/tseaver/gcloud-python/blob/streaming-strip_unused_features/gcloud/streaming/http_wrapper.py#L225-L276) is seriously awkward, and likely a bug magnet.



Discussion:
As [mentioned](https://github.com/GoogleCloudPlatform/gcloud-python/pull/1210/files#r45370100) in #1210, this should resolve the bare exception clauses as well.


Resolving patch:
diff --git a/gcloud/streaming/http_wrapper.py b/gcloud/streaming/http_wrapper.py
--- a/gcloud/streaming/http_wrapper.py
+++ b/gcloud/streaming/http_wrapper.py
@@ -24,6 +24,8 @@
 # 308 and 429 don't have names in httplib.
 RESUME_INCOMPLETE = 308
 TOO_MANY_REQUESTS = 429
+
+
 _REDIRECT_STATUS_CODES = (
     http_client.MOVED_PERMANENTLY,
     http_client.FOUND,
@@ -33,6 +35,19 @@
 )
 
 
+_RETRYABLE_EXCEPTIONS = (
+    http_client.BadStatusLine,
+    http_client.IncompleteRead,
+    http_client.ResponseNotReady,
+    socket.error,
+    httplib2.ServerNotFoundError,
+    ValueError,
+    RequestError,
+    BadStatusCodeError,
+    RetryAfterError,
+)
+
+
 class _ExceptionRetryArgs(
         collections.namedtuple(
             '_ExceptionRetryArgs',
@@ -276,7 +291,7 @@ def _check_response(response):
         raise RetryAfterError.from_response(response)
 
 
-def _rebuild_http_connections(http):
+def _reset_http_connections(http):
     """Rebuild all http connections in the httplib2.Http instance.
 
     httplib2 overloads the map in http.connections to contain two different
@@ -295,60 +310,6 @@ def _rebuild_http_connections(http):
                 del http.connections[conn_key]
 
 
-def handle_http_exceptions(retry_args):
-    """Exception handler for http failures.
-
-    Catches known failures and rebuild the underlying HTTP connections.
-
-    :type retry_args: :class:`_ExceptionRetryArgs`
-    :param retry_args: the exception information to be evaluated.
-    """
-    # If the server indicates how long to wait, use that value.  Otherwise,
-    # calculate the wait time on our own.
-    retry_after = None
-
-    # Transport failures
-    if isinstance(retry_args.exc, (http_client.BadStatusLine,
-                                   http_client.IncompleteRead,
-                                   http_client.ResponseNotReady)):
-        logging.debug('Caught HTTP error %s, retrying: %s',
-                      type(retry_args.exc).__name__, retry_args.exc)
-    elif isinstance(retry_args.exc, socket.gaierror):
-        logging.debug(
-            'Caught socket address error, retrying: %s', retry_args.exc)
-    elif isinstance(retry_args.exc, socket.timeout):
-        logging.debug(
-            'Caught socket timeout error, retrying: %s', retry_args.exc)
-    elif isinstance(retry_args.exc, socket.error):
-        logging.debug('Caught socket error, retrying: %s', retry_args.exc)
-    elif isinstance(retry_args.exc, httplib2.ServerNotFoundError):
-        logging.debug(
-            'Caught server not found error, retrying: %s', retry_args.exc)
-    elif isinstance(retry_args.exc, ValueError):
-        # oauth2client tries to JSON-decode the response, which can result
-        # in a ValueError if the response was invalid. Until that is fixed in
-        # oauth2client, need to handle it here.
-        logging.debug('Response content was invalid (%s), retrying',
-                      retry_args.exc)
-    elif isinstance(retry_args.exc, RequestError):
-        logging.debug('Request returned no response, retrying')
-    # API-level failures
-    elif isinstance(retry_args.exc, BadStatusCodeError):
-        logging.debug('Response returned status %s, retrying',
-                      retry_args.exc.status_code)
-    elif isinstance(retry_args.exc, RetryAfterError):
-        logging.debug('Response returned a retry-after header, retrying')
-        retry_after = retry_args.exc.retry_after
-    else:
-        raise
-    _rebuild_http_connections(retry_args.http)
-    logging.debug('Retrying request to url %s after exception %s',
-                  retry_args.http_request.url, retry_args.exc)
-    time.sleep(
-        retry_after or calculate_wait_for_retry(
-            retry_args.num_retries, max_wait=retry_args.max_retry_wait))
-
-
 def _make_api_request_no_retry(http, http_request, redirections=5,
                                check_response_func=_check_response):
     """Send an HTTP request via the given http instance.
@@ -403,7 +364,6 @@ def make_api_request(http, http_request,
                      retries=7,
                      max_retry_wait=60,
                      redirections=5,
-                     retry_func=handle_http_exceptions,
                      check_response_func=_check_response,
                      wo_retry_func=_make_api_request_no_retry):
     """Send an HTTP request via the given http, performing error/retry handling.
@@ -424,9 +384,6 @@ def make_api_request(http, http_request,
     :type redirections: integer
     :param redirections: Number of redirects to follow.
 
-    :type retry_func: function taking (http, request, exception, num_retries).
-    :param retry_func: Function to handle retries on exceptions.
-
     :type check_response_func: function taking (response, content, url).
     :param check_response_func: Function to validate the HTTP response.
 
@@ -446,14 +403,18 @@ def make_api_request(http, http_request,
             return wo_retry_func(
                 http, http_request, redirections=redirections,
                 check_response_func=check_response_func)
-        # retry_func will consume the exception types it handles and raise.
-        except Exception as exc:  # pylint: disable=broad-except
+        except _RETRYABLE_EXCEPTIONS as exc:
             retry += 1
             if retry >= retries:
                 raise
-            else:
-                retry_func(_ExceptionRetryArgs(
-                    http, http_request, exc, retry, max_retry_wait))
+            retry_after = getattr(exc, 'retry_after', None)
+            if retry_after is None:
+                retry_after = calculate_wait_for_retry(retry, max_retry_wait)
+
+            _reset_http_connections(http)
+            logging.debug('Retrying request to url %s after exception %s',
+                          http_request.url, type(exc).__name__)
+            time.sleep(retry_after)
 
 
 _HTTP_FACTORIES = []
diff --git a/gcloud/streaming/transfer.py b/gcloud/streaming/transfer.py
--- a/gcloud/streaming/transfer.py
+++ b/gcloud/streaming/transfer.py
@@ -15,7 +15,6 @@
 from gcloud.streaming.exceptions import TransferInvalidError
 from gcloud.streaming.exceptions import TransferRetryError
 from gcloud.streaming.http_wrapper import get_http
-from gcloud.streaming.http_wrapper import handle_http_exceptions
 from gcloud.streaming.http_wrapper import make_api_request
 from gcloud.streaming.http_wrapper import Request
 from gcloud.streaming.http_wrapper import RESUME_INCOMPLETE
@@ -65,7 +64,6 @@ def __init__(self, stream, close_stream=False,
         # Let the @property do validation
         self.num_retries = num_retries
 
-        self.retry_func = handle_http_exceptions
         self.auto_transfer = auto_transfer
         self.chunksize = chunksize
 
@@ -467,8 +465,7 @@ def _get_chunk(self, start, end):
         request = Request(url=self.url)
         self._set_range_header(request, start, end=end)
         return make_api_request(
-            self.bytes_http, request, retry_func=self.retry_func,
-            retries=self.num_retries)
+            self.bytes_http, request, retries=self.num_retries)
 
     def _process_response(self, response):
         """Update attribtes and writing stream, based on response.
@@ -1060,8 +1057,7 @@ def _send_media_request(self, request, end):
                  code from the response indicates an error.
         """
         response = make_api_request(
-            self.bytes_http, request, retry_func=self.retry_func,
-            retries=self.num_retries)
+            self.bytes_http, request, retries=self.num_retries)
         if response.status_code not in (http_client.OK, http_client.CREATED,
                                         RESUME_INCOMPLETE):
             # We want to reset our state to wherever the server left us


Test patch:
diff --git a/gcloud/streaming/test_http_wrapper.py b/gcloud/streaming/test_http_wrapper.py
--- a/gcloud/streaming/test_http_wrapper.py
+++ b/gcloud/streaming/test_http_wrapper.py
@@ -233,11 +233,11 @@ def test_pass(self):
         self._callFUT(_Response(200))
 
 
-class Test__rebuild_http_connections(unittest2.TestCase):
+class Test__reset_http_connections(unittest2.TestCase):
 
     def _callFUT(self, *args, **kw):
-        from gcloud.streaming.http_wrapper import _rebuild_http_connections
-        return _rebuild_http_connections(*args, **kw)
+        from gcloud.streaming.http_wrapper import _reset_http_connections
+        return _reset_http_connections(*args, **kw)
 
     def test_wo_connections(self):
         http = object()
@@ -251,248 +251,6 @@ def test_w_connections(self):
         self.assertTrue('skip_me' in connections)
 
 
-class Test_handle_http_exceptions(unittest2.TestCase):
-    URL = 'http://example.com/api'
-
-    def _callFUT(self, *args, **kw):
-        from gcloud.streaming.http_wrapper import (
-            handle_http_exceptions)
-        return handle_http_exceptions(*args, **kw)
-
-    def _monkeyMUT(self):
-        from gcloud._testing import _Monkey
-        from gcloud.streaming import http_wrapper as MUT
-        _logged = []
-
-        def _debug(msg, *args):
-            _logged.append((msg, args))
-
-        _logging = _Dummy(debug=_debug)
-        _slept = []
-
-        def _sleep(value):
-            _slept.append(value)
-
-        _time = _Dummy(sleep=_sleep)
-        monkey = _Monkey(MUT, logging=_logging, time=_time)
-        return monkey, _logged, _slept
-
-    def _build_retry_args(self, exc,
-                          url=URL, num_retries=0, max_retry_wait=10):
-        retry_args = _Dummy(exc=exc,
-                            num_retries=num_retries,
-                            max_retry_wait=max_retry_wait)
-        retry_args.http_request = _Dummy(url=url)
-        connections = {'delete:me': object(), 'skip_me': object()}
-        retry_args.http = _Dummy(connections=connections)
-        return retry_args
-
-    def _verify_logged_slept(self, logged, slept,
-                             expected_msg, expected_args, expected_sleep=1.25):
-        self.assertEqual(len(logged), 2)
-
-        msg, args = logged[0]
-        self.assertEqual(msg, expected_msg)
-        self.assertEqual(args, expected_args)
-
-        msg, args = logged[1]
-        self.assertEqual(msg, 'Retrying request to url %s after exception %s')
-        self.assertEqual(len(args), 2)
-        self.assertEqual(args[0], self.URL)
-
-        self.assertEqual(slept, [expected_sleep])
-
-    def test_w_BadStatusLine(self):
-        import random
-        from gcloud._testing import _Monkey
-        from six.moves.http_client import BadStatusLine
-        exc = BadStatusLine('invalid')
-        retry_args = self._build_retry_args(exc)
-        monkey, logged, slept = self._monkeyMUT()
-
-        with _Monkey(random, uniform=lambda lower, upper: upper):
-            with monkey:
-                self._callFUT(retry_args)
-
-        self._verify_logged_slept(
-            logged, slept, 'Caught HTTP error %s, retrying: %s',
-            ('BadStatusLine', exc))
-
-    def test_w_IncompleteRead(self):
-        import random
-        from gcloud._testing import _Monkey
-        from six.moves.http_client import IncompleteRead
-        exc = IncompleteRead(50, 100)
-        retry_args = self._build_retry_args(exc)
-        monkey, logged, slept = self._monkeyMUT()
-
-        with _Monkey(random, uniform=lambda lower, upper: upper):
-            with monkey:
-                self._callFUT(retry_args)
-
-        self._verify_logged_slept(
-            logged, slept, 'Caught HTTP error %s, retrying: %s',
-            ('IncompleteRead', exc))
-
-    def test_w_ResponseNotReady(self):
-        import random
-        from gcloud._testing import _Monkey
-        from six.moves.http_client import ResponseNotReady
-        exc = ResponseNotReady('uh oh')
-        retry_args = self._build_retry_args(exc)
-        monkey, logged, slept = self._monkeyMUT()
-
-        with _Monkey(random, uniform=lambda lower, upper: upper):
-            with monkey:
-                self._callFUT(retry_args)
-
-        self._verify_logged_slept(
-            logged, slept, 'Caught HTTP error %s, retrying: %s',
-            ('ResponseNotReady', exc))
-
-    def test_w_socket_gaierror(self):
-        import random
-        from gcloud._testing import _Monkey
-        import socket
-        exc = socket.gaierror('uh oh')
-        retry_args = self._build_retry_args(exc)
-        monkey, logged, slept = self._monkeyMUT()
-
-        with _Monkey(random, uniform=lambda lower, upper: upper):
-            with monkey:
-                self._callFUT(retry_args)
-
-        self._verify_logged_slept(
-            logged, slept, 'Caught socket address error, retrying: %s', (exc,))
-
-    def test_w_socket_timeout(self):
-        import random
-        from gcloud._testing import _Monkey
-        import socket
-        exc = socket.timeout('uh oh')
-        retry_args = self._build_retry_args(exc)
-        monkey, logged, slept = self._monkeyMUT()
-
-        with _Monkey(random, uniform=lambda lower, upper: upper):
-            with monkey:
-                self._callFUT(retry_args)
-
-        self._verify_logged_slept(
-            logged, slept, 'Caught socket timeout error, retrying: %s', (exc,))
-
-    def test_w_socket_error(self):
-        import random
-        from gcloud._testing import _Monkey
-        import socket
-        exc = socket.error('uh oh')
-        retry_args = self._build_retry_args(exc)
-        monkey, logged, slept = self._monkeyMUT()
-
-        with _Monkey(random, uniform=lambda lower, upper: upper):
-            with monkey:
-                self._callFUT(retry_args)
-
-        self._verify_logged_slept(
-            logged, slept, 'Caught socket error, retrying: %s', (exc,))
-
-    def test_w_httplib2_ServerNotFoundError(self):
-        import random
-        from gcloud._testing import _Monkey
-        import httplib2
-        exc = httplib2.ServerNotFoundError('uh oh')
-        retry_args = self._build_retry_args(exc)
-        monkey, logged, slept = self._monkeyMUT()
-
-        with _Monkey(random, uniform=lambda lower, upper: upper):
-            with monkey:
-                self._callFUT(retry_args)
-
-        self._verify_logged_slept(
-            logged, slept,
-            'Caught server not found error, retrying: %s', (exc,))
-
-    def test_w_ValueError(self):
-        import random
-        from gcloud._testing import _Monkey
-        exc = ValueError('uh oh')
-        retry_args = self._build_retry_args(exc)
-        monkey, logged, slept = self._monkeyMUT()
-
-        with _Monkey(random, uniform=lambda lower, upper: upper):
-            with monkey:
-                self._callFUT(retry_args)
-
-        self._verify_logged_slept(
-            logged, slept,
-            'Response content was invalid (%s), retrying', (exc,))
-
-    def test_w_RequestError(self):
-        import random
-        from gcloud._testing import _Monkey
-        from gcloud.streaming.exceptions import RequestError
-        exc = RequestError('uh oh')
-        retry_args = self._build_retry_args(exc)
-        monkey, logged, slept = self._monkeyMUT()
-
-        with _Monkey(random, uniform=lambda lower, upper: upper):
-            with monkey:
-                self._callFUT(retry_args)
-
-        self._verify_logged_slept(
-            logged, slept, 'Request returned no response, retrying', ())
-
-    def test_w_BadStatusCodeError(self):
-        import random
-        from gcloud._testing import _Monkey
-        from gcloud.streaming.exceptions import BadStatusCodeError
-        response = _Response(500)
-        exc = BadStatusCodeError.from_response(response)
-        retry_args = self._build_retry_args(exc)
-        monkey, logged, slept = self._monkeyMUT()
-
-        with _Monkey(random, uniform=lambda lower, upper: upper):
-            with monkey:
-                self._callFUT(retry_args)
-
-        self._verify_logged_slept(
-            logged, slept, 'Response returned status %s, retrying', (500,))
-
-    def test_w_RetryAfterError(self):
-        from gcloud.streaming.exceptions import RetryAfterError
-        from gcloud.streaming.http_wrapper import TOO_MANY_REQUESTS
-        RETRY_AFTER = 25
-        response = _Response(TOO_MANY_REQUESTS, RETRY_AFTER)
-        exc = RetryAfterError.from_response(response)
-        retry_args = self._build_retry_args(exc)
-        monkey, logged, slept = self._monkeyMUT()
-
-        with monkey:
-            self._callFUT(retry_args)
-
-        self._verify_logged_slept(
-            logged, slept,
-            'Response returned a retry-after header, retrying', (),
-            RETRY_AFTER)
-
-    def test_wo_matching_type(self):
-
-        class _Nonesuch(Exception):
-            pass
-
-        def _raises():
-            raise _Nonesuch
-
-        monkey, _, _ = self._monkeyMUT()
-
-        with monkey:
-            with self.assertRaises(_Nonesuch):
-                try:
-                    _raises()
-                except _Nonesuch as exc:
-                    retry_args = _Dummy(exc=exc)
-                    self._callFUT(retry_args)
-
-
 class Test___make_api_request_no_retry(unittest2.TestCase):
 
     def _callFUT(self, *args, **kw):
@@ -635,28 +393,23 @@ def _wo_exception(*args, **kw):
         self.assertEqual(_checked, [])  # not called by '_wo_exception'
 
     def test_w_exceptions_lt_max_retries(self):
-        HTTP, REQUEST, RESPONSE = object(), object(), object()
+        from gcloud.streaming.exceptions import RetryAfterError
+        HTTP, RESPONSE = object(), object()
+        REQUEST = _Request()
         WAIT = 10,
-        _created, _checked, _retried = [], [], []
+        _created, _checked = [], []
         _counter = [None] * 4
 
-        class _Retry(Exception):
-            pass
-
         def _wo_exception(*args, **kw):
             _created.append((args, kw))
             if _counter:
                 _counter.pop()
-                raise _Retry()
+                raise RetryAfterError(RESPONSE, '', REQUEST.url, 0.1)
             return RESPONSE
 
-        def _retry(args):
-            _retried.append(args)
-
         response = self._callFUT(HTTP, REQUEST,
                                  retries=5,
                                  max_retry_wait=WAIT,
-                                 retry_func=_retry,
                                  wo_retry_func=_wo_exception,
                                  check_response_func=_checked.append)
 
@@ -669,36 +422,26 @@ def _retry(args):
         for attempt in _created:
             self.assertEqual(attempt, ((HTTP, REQUEST), expected_kw))
         self.assertEqual(_checked, [])  # not called by '_wo_exception'
-        self.assertEqual(len(_retried), 4)
-        for index, retry in enumerate(_retried):
-            self.assertTrue(retry.http is HTTP)
-            self.assertTrue(retry.http_request is REQUEST)
-            self.assertTrue(isinstance(retry.exc, _Retry))
-            self.assertEqual(retry.num_retries, index + 1)
-            self.assertEqual(retry.max_retry_wait, WAIT)
 
     def test_w_exceptions_gt_max_retries(self):
-        HTTP, REQUEST = object(), object()
+        from gcloud._testing import _Monkey
+        from gcloud.streaming import http_wrapper as MUT
+        HTTP = object()
+        REQUEST = _Request()
         WAIT = 10,
-        _created, _checked, _retried = [], [], []
-
-        class _Retry(Exception):
-            pass
+        _created, _checked = [], []
 
         def _wo_exception(*args, **kw):
             _created.append((args, kw))
-            raise _Retry()
-
-        def _retry(args):
-            _retried.append(args)
+            raise ValueError('Retryable')
 
-        with self.assertRaises(_Retry):
-            self._callFUT(HTTP, REQUEST,
-                          retries=3,
-                          max_retry_wait=WAIT,
-                          retry_func=_retry,
-                          wo_retry_func=_wo_exception,
-                          check_response_func=_checked.append)
+        with _Monkey(MUT, calculate_wait_for_retry=lambda *ignored: 0.1):
+            with self.assertRaises(ValueError):
+                self._callFUT(HTTP, REQUEST,
+                              retries=3,
+                              max_retry_wait=WAIT,
+                              wo_retry_func=_wo_exception,
+                              check_response_func=_checked.append)
 
         self.assertEqual(len(_created), 3)
         expected_kw = {
@@ -708,13 +451,6 @@ def _retry(args):
         for attempt in _created:
             self.assertEqual(attempt, ((HTTP, REQUEST), expected_kw))
         self.assertEqual(_checked, [])  # not called by '_wo_exception'
-        self.assertEqual(len(_retried), 2)
-        for index, retry in enumerate(_retried):
-            self.assertTrue(retry.http is HTTP)
-            self.assertTrue(retry.http_request is REQUEST)
-            self.assertTrue(isinstance(retry.exc, _Retry))
-            self.assertEqual(retry.num_retries, index + 1)
-            self.assertEqual(retry.max_retry_wait, WAIT)
 
 
 class Test__register_http_factory(unittest2.TestCase):