Repository: dwavesystems/dwave-cloud-client

Issue:
Retry SAPI requests whenever possible
We can (and should) easily retry:
- idempotent requests
- socket connect errors (including connect timeout)
- dns-caused errors

We should not retry on:
- non-idempotent requests (like problem submit). Maybe allow some requests to be explicitly retried (like initiate problem upload)
- client/user-caused errors
- most internal server errors
- certificate errors

Discussion:
randomir: Far from what I plan to do under this banner, but I'll break it up in a few PRs.

Patch:
diff --git a/docs/conf.py b/docs/conf.py
index 0cdd555..3949505 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -105,6 +105,7 @@ InstanceAttributeDocumenter.add_directive_header = iad_add_directive_header
 intersphinx_mapping = {
     'python': ('https://docs.python.org/3', None),
     'numpy': ('https://numpy.org/doc/stable/', None),
+    'urllib3': ('https://urllib3.readthedocs.io/en/stable/', None),
     'oceandocs': ('https://docs.ocean.dwavesys.com/en/stable/', None),
     'sysdocs_gettingstarted': ('https://docs.dwavesys.com/docs/latest/', None),
 }
diff --git a/dwave/cloud/client.py b/dwave/cloud/client.py
index 4744bf9..341728f 100644
--- a/dwave/cloud/client.py
+++ b/dwave/cloud/client.py
@@ -62,20 +62,21 @@ from collections import abc, namedtuple, OrderedDict
 from concurrent.futures import ThreadPoolExecutor
 
 import requests
+import urllib3
 from dateutil.parser import parse as parse_datetime
 from plucky import pluck
 
 from dwave.cloud.package_info import __packagename__, __version__
 from dwave.cloud.exceptions import *
 from dwave.cloud.computation import Future
-from dwave.cloud.config import load_config, parse_float, parse_boolean
+from dwave.cloud.config import load_config, parse_float, parse_int, parse_boolean
 from dwave.cloud.solver import Solver, available_solvers
 from dwave.cloud.concurrency import PriorityThreadPoolExecutor
 from dwave.cloud.upload import ChunkedData
 from dwave.cloud.events import dispatch_event
 from dwave.cloud.utils import (
     TimeoutingHTTPAdapter, BaseUrlSession, user_agent,
-    datetime_to_timestamp, utcnow, epochnow, cached, retried)
+    datetime_to_timestamp, utcnow, epochnow, cached, retried, is_caused_by)
 
 __all__ = ['Client']
 
@@ -142,6 +143,40 @@ class Client(object):
             Problem status is polled with exponential back-off schedule.
             Maximum back-off period is limited to ``poll_backoff_max`` seconds.
 
+        http_retry_total (int, default=10):
+            Total number of retries of failing idempotent HTTP requests to
+            allow. Takes precedence over other counts.
+            See ``total`` in :class:`~urllib3.util.retry.Retry` for details.
+
+        http_retry_connect (int, default=None):
+            How many connection-related errors to retry on.
+            See ``connect`` in :class:`~urllib3.util.retry.Retry` for details.
+
+        http_retry_read (int, default=None):
+            How many times to retry on read errors.
+            See ``read`` in :class:`~urllib3.util.retry.Retry` for details.
+
+        http_retry_redirect (int, default=None):
+            How many redirects to perform.
+            See ``redirect`` in :class:`~urllib3.util.retry.Retry` for details.
+
+        http_retry_status (int, default=None):
+            How many times to retry on bad status codes.
+            See ``status`` in :class:`~urllib3.util.retry.Retry` for details.
+
+        http_retry_backoff_factor (float, default=0.01):
+            A backoff factor to apply between attempts after the second try.
+            Sleep between retries, in seconds::
+
+                {backoff factor} * (2 ** ({number of total retries} - 1))
+
+            See ``backoff_factor`` in :class:`~urllib3.util.retry.Retry` for
+            details.
+
+        http_retry_backoff_max (float, default=60):
+            Maximum backoff time in seconds.
+            See :attr:`~urllib3.util.retry.Retry.BACKOFF_MAX` for details.
+
         defaults (dict, optional):
             Defaults for the client instance that override the class
             :attr:`.DEFAULTS`.
@@ -199,6 +234,14 @@ class Client(object):
         # poll back-off schedule defaults [sec]
         'poll_backoff_min': 0.05,
         'poll_backoff_max': 60,
+        # idempotent http requests retry params
+        'http_retry_total': 10,
+        'http_retry_connect': None,
+        'http_retry_read': None,
+        'http_retry_redirect': None,
+        'http_retry_status': None,
+        'http_retry_backoff_factor': 0.01,
+        'http_retry_backoff_max': 60,
     }
 
     # Number of problems to include in a submit/status query
@@ -325,9 +368,12 @@ class Client(object):
 
         # combine instance-level defaults with file/env/kwarg option values
         # note: treat empty string values (e.g. from file/env) as undefined/None
+        # TODO: to avoid the `first_non_empty` hack, we need to parse config
+        # values downstream (immediately after reading from file or env var)
+        first_non_empty = lambda a, b: b if a == '' or a is None else a
         options = {}
         for k, v in self.defaults.items():
-            options[k] = kwargs.get(k) or v
+            options[k] = first_non_empty(kwargs.get(k), v)
 
         logger.debug("Client options with defaults: %r", options)
 
@@ -391,6 +437,7 @@ class Client(object):
         logger.debug("Parsed headers=%r", headers_dict)
 
         # Store connection/session parameters
+        # TODO: consolidate all options under Client.options or similar
         self.endpoint = endpoint
         self.token = token
         self.default_solver = solver_def
@@ -407,21 +454,29 @@ class Client(object):
         self.poll_backoff_min = parse_float(options['poll_backoff_min'])
         self.poll_backoff_max = parse_float(options['poll_backoff_max'])
 
+        self.http_retry_total = parse_int(options['http_retry_total'])
+        self.http_retry_connect = parse_int(options['http_retry_connect'])
+        self.http_retry_read = parse_int(options['http_retry_read'])
+        self.http_retry_redirect = parse_int(options['http_retry_redirect'])
+        self.http_retry_status = parse_int(options['http_retry_status'])
+        self.http_retry_backoff_factor = parse_float(options['http_retry_backoff_factor'])
+        self.http_retry_backoff_max = parse_float(options['http_retry_backoff_max'])
+
+        opts = (
+            'endpoint', 'token', 'default_solver',
+            'client_cert', 'request_timeout', 'polling_timeout',
+            'proxy', 'headers', 'permissive_ssl', 'connection_close',
+            'poll_backoff_min', 'poll_backoff_max',
+            'http_retry_total', 'http_retry_connect', 'http_retry_read',
+            'http_retry_redirect', 'http_retry_status',
+            'http_retry_backoff_factor', 'http_retry_backoff_max')
+        logger.debug(
+            "Client initialized with (%s)",
+            ", ".join("{}={!r}".format(o, getattr(self, o)) for o in opts))
+
         # Create session for main thread only
         self.session = self.create_session()
 
-        logger.debug(
-            "Client initialized with ("
-            "endpoint=%r, token=%r, default_solver=%r, proxy=%r, "
-            "permissive_ssl=%r, request_timeout=%r, polling_timeout=%r, "
-            "connection_close=%r, headers=%r, client_cert=%r, "
-            "poll_backoff_min=%r, poll_backoff_max=%r)",
-            self.endpoint, self.token, self.default_solver, self.proxy,
-            self.permissive_ssl, self.request_timeout, self.polling_timeout,
-            self.connection_close, self.headers, self.client_cert,
-            self.poll_backoff_min, self.poll_backoff_max
-        )
-
         # Build the problem submission queue, start its workers
         self._submission_queue = queue.Queue()
         self._submission_workers = []
@@ -483,9 +538,31 @@ class Client(object):
         if not endpoint.endswith('/'):
             endpoint += '/'
 
+        # create http idempotent Retry config
+        def get_retry_conf():
+
+            # need a subclass to override the backoff_max
+            class Retry(urllib3.Retry):
+                BACKOFF_MAX = self.http_retry_backoff_max
+
+            return Retry(
+                total=self.http_retry_total,
+                connect=self.http_retry_connect,
+                read=self.http_retry_read,
+                redirect=self.http_retry_redirect,
+                status=self.http_retry_status,
+                backoff_factor=self.http_retry_backoff_factor,
+                raise_on_redirect=True,
+                raise_on_status=True,
+                respect_retry_after_header=True)
+
         session = BaseUrlSession(base_url=endpoint)
-        session.mount('http://', TimeoutingHTTPAdapter(timeout=self.request_timeout))
-        session.mount('https://', TimeoutingHTTPAdapter(timeout=self.request_timeout))
+        session.mount('http://',
+            TimeoutingHTTPAdapter(timeout=self.request_timeout,
+                                  max_retries=get_retry_conf()))
+        session.mount('https://',
+            TimeoutingHTTPAdapter(timeout=self.request_timeout,
+                                  max_retries=get_retry_conf()))
 
         session.headers.update({'User-Agent': user_agent(__packagename__, __version__)})
         if self.headers:
@@ -603,18 +680,12 @@ class Client(object):
             url = 'solvers/remote/'
 
         try:
-            response = self.session.get(url)
-        except requests.exceptions.Timeout:
-            raise RequestTimeout
-
-        if response.status_code == 401:
-            raise SolverAuthenticationError
-
-        if name is not None and response.status_code == 404:
-            raise SolverNotFoundError("No solver with name={!r} available".format(name))
-
-        response.raise_for_status()
-        data = response.json()
+            data = Client._sapi_request(self.session.get, url)
+        except SAPIError as exc:
+            if name is not None and exc.error_code == 404:
+                raise SolverNotFoundError("No solver with name={!r} available".format(name))
+            else:
+                raise
 
         if name is not None:
             data = [data]
@@ -1161,38 +1232,24 @@ class Client(object):
                 # Submit the problems
                 logger.debug("Submitting %d problems", len(ready_problems))
                 try:
-                    body = '[' + ','.join(mess.body.result() for mess in ready_problems) + ']'
-                    try:
-                        response = session.post('problems/', body)
-                        localtime_of_response = epochnow()
-                    except requests.exceptions.Timeout:
-                        raise RequestTimeout
-
-                    if response.status_code == 401:
-                        raise SolverAuthenticationError()
-                    response.raise_for_status()
-
-                    message = response.json()
+                    body = '[' + ','.join(msg.body.result() for msg in ready_problems) + ']'
+                    logger.debug('Size of POST body = %d', len(body))
+                    message = Client._sapi_request(session.post, 'problems/', body)
                     logger.debug("Finished submitting %d problems", len(ready_problems))
-                except BaseException as exception:
-                    logger.debug("Submit failed for %d problems", len(ready_problems))
-                    if not isinstance(exception, SolverAuthenticationError):
-                        exception = IOError(exception)
 
-                    for mess in ready_problems:
-                        mess.future._set_exception(exception)
+                except Exception as exc:
+                    logger.debug("Submit failed for %d problems with %r",
+                                 len(ready_problems), exc)
+                    for msg in ready_problems:
+                        msg.future._set_exception(exc)
                         task_done()
                     continue
 
                 # Pass on the information
                 for submission, res in zip(ready_problems, message):
-                    submission.future._set_clock_diff(response, localtime_of_response)
                     self._handle_problem_status(res, submission.future)
                     task_done()
 
-                # this is equivalent to a yield to scheduler in other threading libraries
-                time.sleep(0)
-
         except BaseException as err:
             logger.exception(err)
 
@@ -1328,12 +1385,8 @@ class Client(object):
                 # Submit the problems, attach the ids as a json list in the
                 # body of the delete query.
                 try:
-                    body = [item[0] for item in item_list]
-
-                    try:
-                        session.delete('problems/', json=body)
-                    except requests.exceptions.Timeout:
-                        raise RequestTimeout
+                    ids = [item[0] for item in item_list]
+                    Client._sapi_request(session.delete, 'problems/', json=ids)
 
                 except Exception as exc:
                     for _, future in item_list:
@@ -1341,10 +1394,8 @@ class Client(object):
                             future._set_exception(exc)
 
                 # Mark all the ids as processed regardless of success or failure.
-                [self._cancel_queue.task_done() for _ in item_list]
-
-                # this is equivalent to a yield to scheduler in other threading libraries
-                time.sleep(0)
+                for _ in item_list:
+                    self._cancel_queue.task_done()
 
         except Exception as err:
             logger.exception(err)
@@ -1457,46 +1508,35 @@ class Client(object):
                     logger.trace("Executing poll API request")
 
                     try:
-                        response = session.get(query_string)
-                    except requests.exceptions.Timeout:
-                        raise RequestTimeout
-
-                    if response.status_code == 401:
-                        raise SolverAuthenticationError()
-
-                    # assume 5xx errors are transient, and don't abort polling
-                    if 500 <= response.status_code < 600:
-                        logger.warning(
-                            "Received an internal server error response on "
-                            "problem status polling request (%s). Assuming "
-                            "error is transient, and resuming polling.",
-                            response.status_code)
-                        # add all futures in this frame back to the polling queue
-                        # XXX: logic split between `_handle_problem_status` and here
-                        for future in frame_futures.values():
-                            self._poll(future)
+                        statuses = Client._sapi_request(session.get, query_string)
+
+                    except SAPIError as exc:
+                        # assume 5xx errors are transient, and don't abort polling
+                        if 500 <= exc.error_code < 600:
+                            logger.warning(
+                                "Received an internal server error response on "
+                                "problem status polling request (%s). Assuming "
+                                "error is transient, and resuming polling.",
+                                exc.error_code)
+                            # add all futures in this frame back to the polling queue
+                            # XXX: logic split between `_handle_problem_status` and here
+                            for future in frame_futures.values():
+                                self._poll(future)
+                        else:
+                            raise
 
                     else:
-                        # otherwise, fail
-                        response.raise_for_status()
-
-                        # or handle a successful request
-                        statuses = response.json()
+                        # handle a successful request
                         for status in statuses:
                             self._handle_problem_status(status, frame_futures[status['id']])
 
-                except BaseException as exception:
-                    if not isinstance(exception, SolverAuthenticationError):
-                        exception = IOError(exception)
-
+                except Exception as exc:
                     for id_ in frame_futures.keys():
-                        frame_futures[id_]._set_exception(exception)
+                        frame_futures[id_]._set_exception(exc)
 
                 for id_ in frame_futures.keys():
                     task_done()
 
-                time.sleep(0)
-
         except Exception as err:
             logger.exception(err)
 
@@ -1534,32 +1574,13 @@ class Client(object):
 
                 # Submit the query
                 query_string = 'problems/{}/'.format(future.id)
-                try:
-                    logger.trace("Executing answer load request")
-
-                    try:
-                        response = session.get(query_string)
-                    except requests.exceptions.Timeout:
-                        raise RequestTimeout
-
-                    logger.trace(("Answer load response: "
-                                  "code={r.status_code!r}, "
-                                  "body={r.text!r}").format(r=response))
 
-                    if response.status_code == 401:
-                        raise SolverAuthenticationError()
-                    if response.status_code == 404:
-                        raise SolverError(response.text)
-                    response.raise_for_status()
-
-                    message = response.json()
-                except BaseException as exception:
-                    logger.trace("Answer load request failed with %r", exception)
-
-                    if not isinstance(exception, SolverError):
-                        exception = IOError(exception)
+                try:
+                    message = Client._sapi_request(session.get, query_string)
 
-                    future._set_exception(exception)
+                except Exception as exc:
+                    logger.debug("Answer load request failed with %r", exc)
+                    future._set_exception(exc)
                     self._load_queue.task_done()
                     continue
 
@@ -1567,9 +1588,6 @@ class Client(object):
                 self._handle_problem_status(message, future)
                 self._load_queue.task_done()
 
-                # this is equivalent to a yield to scheduler in other threading libraries
-                time.sleep(0)
-
         except Exception as err:
             logger.error('Load result error: ' + str(err))
 
@@ -1600,29 +1618,75 @@ class Client(object):
             self._upload_problem_worker, problem=problem, problem_id=problem_id)
 
     @staticmethod
-    def _parse_multipart_upload_response(response):
-        """Parse the JSON response body, raising appropriate exceptions."""
+    def _sapi_request(meth, *args, **kwargs):
+        """Execute an HTTP request defined with the ``meth`` callable and
+        parse the response and interpret errors in compliance with SAPI REST
+        interface.
+
+        Note:
+            For internal use only.
+
+        Args:
+            meth (callable):
+                Callable object to be called with args and kwargs supplied, with
+                expected behavior consistent with one of ``requests.Session()``
+                request methods.
+
+            *args, **kwargs (list, dict):
+                Arguments to the ``meth`` callable.
+
+        Returns:
+            dict: JSON decoded body.
+
+        Raises:
+            A :class:`~dwave.cloud.exceptions.SAPIError` subclass, or
+            :class:`~dwave.cloud.exceptions.RequestTimeout`.
+        """
 
         caller = inspect.stack()[1].function
-        logger.trace("%s response: (code=%r, text=%r)",
+        verb = meth.__name__
+        logger.trace("[%s] request: session.%s(*%r, **%r)", caller, verb, args, kwargs)
+
+        # execute request
+        try:
+            response = meth(*args, **kwargs)
+        except Exception as exc:
+            if is_caused_by(exc, (requests.exceptions.Timeout,
+                                  urllib3.exceptions.TimeoutError)):
+                raise RequestTimeout
+            else:
+                raise
+
+        # parse response
+        logger.trace("[%s] response: (code=%r, body=%r)",
                      caller, response.status_code, response.text)
 
-        if response.status_code == 401:
-            raise SolverAuthenticationError()
+        # NOTE: the expected behavior is for SAPI to return JSON error on
+        # failure. However, that is currently not the case. We need to work
+        # around this until it's fixed.
 
-        try:
-            msg = response.json()
-        except:
-            response.raise_for_status()
+        # no error -> body is json
+        # error -> body can be json or plain text error message
+        if response.ok:
+            try:
+                return response.json()
+            except:
+                raise InvalidAPIResponseError("JSON response expected")
+
+        else:
+            if response.status_code == 401:
+                raise SolverAuthenticationError(error_code=401)
 
-        if response.status_code != 200:
             try:
+                msg = response.json()
                 error_msg = msg['error_msg']
-            except KeyError:
-                response.raise_for_status()
-            raise ProblemUploadError(error_msg)
+                error_code = msg['error_code']
+            except:
+                error_msg = response.text
+                error_code = response.status_code
 
-        return msg
+            # NOTE: for backwards compat only. Change to: SAPIError
+            raise SolverError(error_msg=error_msg, error_code=error_code)
 
     @staticmethod
     @retried(_UPLOAD_REQUEST_RETRIES, backoff=_UPLOAD_RETRIES_BACKOFF)
@@ -1633,14 +1697,7 @@ class Client(object):
 
         path = 'bqm/multipart'
         body = dict(size=size)
-
-        logger.trace("session.post(path=%r, json=%r)", path, body)
-        try:
-            response = session.post(path, json=body)
-        except requests.exceptions.Timeout:
-            raise RequestTimeout
-
-        msg = Client._parse_multipart_upload_response(response)
+        msg = Client._sapi_request(session.post, path, json=body)
 
         try:
             problem_id = msg['id']
@@ -1730,14 +1787,7 @@ class Client(object):
             'Content-Type': 'application/octet-stream',
         }
 
-        logger.trace("session.put(path=%r, data=%r, headers=%r)",
-                     path, part_stream, headers)
-        try:
-            response = session.put(path, data=part_stream, headers=headers)
-        except requests.exceptions.Timeout:
-            raise RequestTimeout
-
-        msg = Client._parse_multipart_upload_response(response)
+        msg = Client._sapi_request(session.put, path, data=part_stream, headers=headers)
 
         logger.debug("Uploaded part_id=%r of problem_id=%r", part_id, problem_id)
 
@@ -1750,13 +1800,7 @@ class Client(object):
 
         path = 'bqm/multipart/{problem_id}/status'.format(problem_id=problem_id)
 
-        logger.trace("session.get(path=%r)", path)
-        try:
-            response = session.get(path)
-        except requests.exceptions.Timeout:
-            raise RequestTimeout
-
-        msg = Client._parse_multipart_upload_response(response)
+        msg = Client._sapi_request(session.get, path)
 
         try:
             msg['status']
@@ -1786,14 +1830,7 @@ class Client(object):
         path = 'bqm/multipart/{problem_id}/combine'.format(problem_id=problem_id)
         body = dict(checksum=checksum)
 
-        logger.trace("session.post(path=%r, json=%r)", path, body)
-
-        try:
-            response = session.post(path, json=body)
-        except requests.exceptions.Timeout:
-            raise RequestTimeout
-
-        msg = Client._parse_multipart_upload_response(response)
+        msg = Client._sapi_request(session.post, path, json=body)
 
         logger.debug("Issued a combine command for problem_id=%r", problem_id)
 
diff --git a/dwave/cloud/config.py b/dwave/cloud/config.py
index 455f086..f44d799 100644
--- a/dwave/cloud/config.py
+++ b/dwave/cloud/config.py
@@ -253,6 +253,20 @@ def parse_float(s, default=None):
     return float(s)
 
 
+def parse_int(s, default=None):
+    """Parse value as returned by ConfigParse as int.
+
+    NB: we need this instead of ``ConfigParser.getint`` when we're parsing
+    values downstream.
+    """
+
+    if s is None or s == '':
+        return default
+    if float(s) != int(s):
+        raise ValueError
+    return int(s)
+
+
 def parse_boolean(s, default=None):
     """Parse value as returned by ConfigParse as bool.
 
diff --git a/dwave/cloud/exceptions.py b/dwave/cloud/exceptions.py
index efac4a5..db47e6c 100644
--- a/dwave/cloud/exceptions.py
+++ b/dwave/cloud/exceptions.py
@@ -22,23 +22,50 @@ class ConfigFileParseError(ConfigFileError):
     """Invalid format of config file."""
 
 
-class SolverError(Exception):
+class SAPIError(Exception):
+    """Generic SAPI error base class."""
+
+    def __init__(self, *args, **kwargs):
+        self.error_msg = kwargs.pop('error_msg', None)
+        self.error_code = kwargs.pop('error_code', None)
+        if len(args) < 1:
+            args = (self.error_msg, )
+        super(SAPIError, self).__init__(*args, **kwargs)
+
+    def __str__(self):
+        return super(SAPIError, self).__str__() or self.error_msg or ''
+
+class ResourceAuthenticationError(SAPIError):
+    """Access to resource not authorized."""
+
+class ResourceNotFoundError(SAPIError):
+    """Resource not found."""
+
+
+class SolverError(SAPIError):
     """Generic base class for all solver-related errors."""
 
+class ProblemError(SAPIError):
+    """Generic base class for all problem-related errors."""
+
+
 class SolverFailureError(SolverError):
     """An exception raised when there is a remote failure calling a solver."""
 
-class SolverNotFoundError(SolverError):
+class SolverNotFoundError(ResourceNotFoundError, SolverError):
     """Solver with matching feature set not found / not available."""
 
+class ProblemNotFoundError(ResourceNotFoundError, ProblemError):
+    """Problem not found."""
+
 class SolverOfflineError(SolverError):
     """Action attempted on an offline solver."""
 
-class SolverAuthenticationError(SolverError):
+class SolverAuthenticationError(ResourceAuthenticationError, SolverError):
     """An exception raised when there is an authentication error."""
 
-    def __init__(self):
-        super(SolverAuthenticationError, self).__init__("Token not accepted for that action.")
+    def __init__(self, *args, **kwargs):
+        super(SolverAuthenticationError, self).__init__("Invalid token or access denied.", *args, **kwargs)
 
 class UnsupportedSolverError(SolverError):
     """The solver we received from the API is not supported by the client."""
@@ -64,7 +91,7 @@ class CanceledFutureError(Exception):
         super(CanceledFutureError, self).__init__("An error occurred reading results from a canceled request")
 
 
-class InvalidAPIResponseError(Exception):
+class InvalidAPIResponseError(SAPIError):
     """Raised when an invalid/unexpected response from D-Wave Solver API is received."""
 
 
diff --git a/dwave/cloud/utils.py b/dwave/cloud/utils.py
index d13018f..1f1a1c4 100644
--- a/dwave/cloud/utils.py
+++ b/dwave/cloud/utils.py
@@ -40,7 +40,8 @@ except ImportError:  # pragma: no cover
 
 __all__ = ['evaluate_ising', 'uniform_iterator', 'uniform_get',
            'default_text_input', 'click_info_switch', 'datetime_to_timestamp',
-           'datetime_to_timestamp', 'utcnow', 'epochnow', 'tictoc']
+           'datetime_to_timestamp', 'utcnow', 'epochnow', 'tictoc',
+           'hasinstance', 'exception_chain', 'is_caused_by']
 
 logger = logging.getLogger(__name__)
 
@@ -303,6 +304,74 @@ class BaseUrlSession(requests.Session):
         return urljoin(self.base_url, url)
 
 
+def hasinstance(iterable, class_or_tuple):
+    """Extension of ``isinstance`` to iterables/sequences. Returns True iff the
+    sequence contains at least one object which is instance of ``class_or_tuple``.
+    """
+
+    return any(isinstance(e, class_or_tuple) for e in iterable)
+
+
+def exception_chain(exception):
+    """Traverse the chain of embedded exceptions, yielding one at the time.
+
+    Args:
+        exception (:class:`Exception`): Chained exception.
+
+    Yields:
+        :class:`Exception`: The next exception in the input exception's chain.
+
+    Examples:
+        def f():
+            try:
+                1/0
+            except ZeroDivisionError:
+                raise ValueError
+
+        try:
+            f()
+        except Exception as e:
+            assert(hasinstance(exception_chain(e), ZeroDivisionError))
+
+    See: PEP-3134.
+    """
+
+    while exception:
+        yield exception
+
+        # explicit exception chaining, i.e `raise .. from ..`
+        if exception.__cause__:
+            exception = exception.__cause__
+
+        # implicit exception chaining
+        elif exception.__context__:
+            exception = exception.__context__
+
+        else:
+            return
+
+
+def is_caused_by(exception, exception_types):
+    """Check if any of ``exception_types`` is causing the ``exception``.
+    Equivalently, check if any of ``exception_types`` is contained in the
+    exception chain rooted at ``exception``.
+
+    Args:
+        exception (:class:`Exception`):
+            Chained exception.
+
+        exception_types (:class:`Exception` or tuple of :class:`Exception`):
+            Exception type or a tuple of exception types to check for.
+
+    Returns:
+        bool:
+            True when ``exception`` is caused by any of the exceptions in
+            ``exception_types``.
+    """
+
+    return hasinstance(exception_chain(exception), exception_types)
+
+
 def user_agent(name=None, version=None):
     """Return User-Agent ~ "name/version language/version interpreter/version os/version"."""
 


Test patch:
diff --git a/tests/requirements.txt b/tests/requirements.txt
index c35d9bd..09c829b 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -1,4 +1,5 @@
 mock
-requests_mock
+requests_mock>=1.8
 coverage
-coveralls
\ No newline at end of file
+coveralls
+parameterized
diff --git a/tests/test_client.py b/tests/test_client.py
index 6c468c2..cf2c299 100644
--- a/tests/test_client.py
+++ b/tests/test_client.py
@@ -25,13 +25,14 @@ import unittest
 from unittest import mock
 from contextlib import contextmanager
 
-import requests.exceptions
+import requests
 from plucky import merge
 
 from dwave.cloud.client import Client
 from dwave.cloud.solver import StructuredSolver, UnstructuredSolver
 from dwave.cloud.exceptions import (
     SolverAuthenticationError, SolverError, SolverNotFoundError)
+from dwave.cloud.testing import iterable_mock_open
 import dwave.cloud
 
 from tests import config
@@ -147,29 +148,29 @@ class SolverLoading(unittest.TestCase):
         with Client(**conf) as base_client:
 
             with qpu.Client(**conf) as client:
-                solvers = base_client.get_solvers(qpu=True)
+                solvers = {s.id for s in base_client.get_solvers(qpu=True)}
                 if solvers:
-                    self.assertEqual(client.get_solver().id, solvers[0].id)
+                    self.assertIn(client.get_solver().id, solvers)
                 else:
                     self.assertRaises(SolverError, client.get_solver)
 
             with sw.Client(**conf) as client:
-                solvers = base_client.get_solvers(software=True)
+                solvers = {s.id for s in base_client.get_solvers(software=True)}
                 if solvers:
-                    self.assertEqual(client.get_solver().id, solvers[0].id)
+                    self.assertIn(client.get_solver().id, solvers)
                 else:
                     self.assertRaises(SolverError, client.get_solver)
 
             with hybrid.Client(**conf) as client:
-                solvers = base_client.get_solvers(hybrid=True)
+                solvers = {s.id for s in base_client.get_solvers(hybrid=True)}
                 if solvers:
-                    self.assertEqual(client.get_solver().id, solvers[0].id)
+                    self.assertIn(client.get_solver().id, solvers)
                 else:
                     self.assertRaises(SolverError, client.get_solver)
 
 
-class ClientFactory(unittest.TestCase):
-    """Test Client.from_config() factory."""
+class ClientConstruction(unittest.TestCase):
+    """Test Client constructor and Client.from_config() factory."""
 
     def test_default(self):
         conf = {k: k for k in 'endpoint token'.split()}
@@ -458,6 +459,105 @@ class ClientFactory(unittest.TestCase):
                 self.assertEqual(client.poll_backoff_min, 0.5)
                 self.assertEqual(client.poll_backoff_max, 1.0)
 
+    def _verify_retry_config(self, retry, opts):
+        self.assertEqual(retry.total, opts['http_retry_total'])
+        self.assertEqual(retry.connect, opts['http_retry_connect'])
+        self.assertEqual(retry.read, opts['http_retry_read'])
+        self.assertEqual(retry.redirect, opts['http_retry_redirect'])
+        self.assertEqual(retry.status, opts['http_retry_status'])
+        self.assertEqual(retry.backoff_factor, opts['http_retry_backoff_factor'])
+        self.assertEqual(retry.BACKOFF_MAX, opts['http_retry_backoff_max'])
+
+    def test_http_retry_params_from_config(self):
+        retry_opts = {
+            "http_retry_total": 3,
+            "http_retry_connect": 2,
+            "http_retry_read": 2,
+            "http_retry_redirect": 0,
+            "http_retry_status": 2,
+            "http_retry_backoff_factor": 0.5,
+            "http_retry_backoff_max": 30,
+        }
+        retry_conf = {k: str(v) for k, v in retry_opts.items()}
+        conf = dict(token='token', **retry_conf)
+
+        # http retry params from config file propagated to client object
+        with mock.patch("dwave.cloud.client.load_config", lambda **kw: conf):
+            with dwave.cloud.Client.from_config() as client:
+                for opt, val in retry_opts.items():
+                    self.assertEqual(getattr(client, opt), val,
+                                     "%s doesn't match" % opt)
+
+                # verify Retry object config
+                retry = client.session.get_adapter('https://').max_retries
+                self._verify_retry_config(retry, retry_opts)
+
+        # test defaults
+        conf = dict(token='token')
+        with mock.patch("dwave.cloud.client.load_config", lambda **kw: conf):
+            with dwave.cloud.Client.from_config() as client:
+                for param in retry_conf:
+                    self.assertEqual(getattr(client, param), Client.DEFAULTS[param])
+
+    def test_http_retry_params_from_kwargs(self):
+        retry_kwargs = {
+            "http_retry_total": 3,
+            "http_retry_connect": 2,
+            "http_retry_read": None,
+            "http_retry_redirect": 0,
+            "http_retry_status": None,
+            "http_retry_backoff_factor": 0.5,
+            "http_retry_backoff_max": 30,
+        }
+        conf = dict(token='token')
+
+        with mock.patch("dwave.cloud.client.load_config", lambda **kw: conf):
+            with dwave.cloud.Client.from_config(**retry_kwargs) as client:
+                # verify client final config
+                for opt, val in retry_kwargs.items():
+                    self.assertEqual(getattr(client, opt), val,
+                                     "%s doesn't match" % opt)
+
+                # verify Retry object config
+                retry = client.session.get_adapter('https://').max_retries
+                self._verify_retry_config(retry, retry_kwargs)
+
+
+class ClientConfigIntegration(unittest.TestCase):
+
+    def test_custom_options(self):
+        """Test custom options (request_timeout, polling_timeout, permissive_ssl)
+        are propagated to Client."""
+
+        request_timeout = 15
+        polling_timeout = 180
+
+        config_body = """
+            [custom]
+            token = 123
+            permissive_ssl = on
+            request_timeout = {}
+            polling_timeout = {}
+        """.format(request_timeout, polling_timeout)
+
+        with mock.patch("dwave.cloud.config.open", iterable_mock_open(config_body)):
+            with Client.from_config('config_file', profile='custom') as client:
+                # check permissive_ssl and timeouts custom params passed-thru
+                self.assertFalse(client.session.verify)
+                self.assertEqual(client.request_timeout, request_timeout)
+                self.assertEqual(client.polling_timeout, polling_timeout)
+
+                # verify client uses those properly
+                def mock_send(*args, **kwargs):
+                    self.assertEqual(kwargs.get('timeout'), request_timeout)
+                    response = requests.Response()
+                    response.status_code = 200
+                    response._content = b'{}'
+                    return response
+
+                with mock.patch("requests.adapters.HTTPAdapter.send", mock_send):
+                    client.get_solvers()
+
 
 class FeatureBasedSolverSelection(unittest.TestCase):
     """Test Client.get_solvers(**filters)."""
diff --git a/tests/test_config.py b/tests/test_config.py
index ccd72f3..88aad77 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -23,10 +23,10 @@ from dwave.cloud.exceptions import ConfigFileParseError, ConfigFileReadError
 from dwave.cloud.testing import iterable_mock_open
 from dwave.cloud.config import (
     get_configfile_paths, load_config_from_files, load_config,
-    parse_float, parse_boolean)
+    parse_float, parse_int, parse_boolean)
 
 
-class TestConfig(unittest.TestCase):
+class TestConfigParsing(unittest.TestCase):
 
     config_body = """
         [defaults]
@@ -400,6 +400,20 @@ class TestConfigUtils(unittest.TestCase):
         self.assertEqual(parse_float(1.5), 1.5)
         self.assertEqual(parse_float(1), 1.0)
 
+    def test_parse_int(self):
+        self.assertEqual(parse_int(None), None)
+        self.assertEqual(parse_int(''), None)
+        self.assertEqual(parse_int('', default=1), 1)
+
+        with self.assertRaises(ValueError):
+            parse_int('1.5')
+        with self.assertRaises(ValueError):
+            parse_int(1.5)
+
+        self.assertEqual(parse_int(123), 123)
+        self.assertEqual(parse_int(0), 0)
+        self.assertEqual(parse_int(-123), -123)
+
     def test_parse_boolean(self):
         self.assertEqual(parse_boolean(None), None)
         self.assertEqual(parse_boolean(''), None)
diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py
index d9e0454..a37b2da 100644
--- a/tests/test_exceptions.py
+++ b/tests/test_exceptions.py
@@ -23,7 +23,7 @@ class TestExceptions(unittest.TestCase):
         try:
             raise SolverAuthenticationError
         except Exception as e:
-            self.assertEqual(str(e), "Token not accepted for that action.")
+            self.assertEqual(str(e), "Invalid token or access denied.")
 
     def test_canceled_future_error_msg(self):
         try:
diff --git a/tests/test_mock_solver_loading.py b/tests/test_mock_solver_loading.py
index a4db844..edc706f 100644
--- a/tests/test_mock_solver_loading.py
+++ b/tests/test_mock_solver_loading.py
@@ -19,31 +19,27 @@ import json
 import unittest
 from unittest import mock
 
-import requests
 import requests_mock
 
 from dwave.cloud.client import Client, Solver
 from dwave.cloud.qpu import Client as QPUClient
 from dwave.cloud.sw import Client as SoftwareClient
 from dwave.cloud.hybrid import Client as HybridClient
-from dwave.cloud.exceptions import (
-    SolverPropertyMissingError, ConfigFileReadError, ConfigFileParseError,
-    SolverError, SolverNotFoundError)
+from dwave.cloud.exceptions import *
 from dwave.cloud.config import load_config
-from dwave.cloud.testing import iterable_mock_open
 
 
 url = 'https://dwavesys.com'
 token = 'abc123abc123abc123abc123abc123abc123'
-solver_name = 'test_solver'
-second_solver_name = 'test_solver2'
+solver1_name = 'first_solver'
+solver2_name = 'second_solver'
 
 bad_url = 'https://not-a-subdomain.dwavesys.com'
 bad_token = '------------------------'
 
 
 def structured_solver_data(id_, cat='qpu', incomplete=False):
-    """Return string describing a single solver."""
+    """Return data dict describing a single solver."""
     obj = {
         "properties": {
             "supported_problem_types": ["qubo", "ising"],
@@ -61,34 +57,35 @@ def structured_solver_data(id_, cat='qpu', incomplete=False):
     if incomplete:
         del obj['properties']['supported_problem_types']
 
-    return json.dumps(obj)
+    return obj
 
 def solver_object(id_, cat='qpu', incomplete=False):
-    return Solver(client=None, data=json.loads(structured_solver_data(id_, cat, incomplete)))
+    return Solver(client=None, data=structured_solver_data(id_, cat, incomplete))
 
 
 # Define the endpoints
+solver1_url = '{}/solvers/remote/{}/'.format(url, solver1_name)
+solver2_url = '{}/solvers/remote/{}/'.format(url, solver2_name)
 all_solver_url = '{}/solvers/remote/'.format(url)
-solver1_url = '{}/solvers/remote/{}/'.format(url, solver_name)
-solver2_url = '{}/solvers/remote/{}/'.format(url, second_solver_name)
 
 
 def setup_server(m):
     """Add endpoints to the server."""
-    # Content strings
-    first_solver_response = structured_solver_data(solver_name)
-    second_solver_response = structured_solver_data(second_solver_name)
-    two_solver_response = '[' + first_solver_response + ',' + second_solver_response + ']'
+
+    solver1_data = structured_solver_data(solver1_name)
+    solver2_data = structured_solver_data(solver2_name)
+    all_solver_data = [solver1_data, solver2_data]
 
     # Setup the server
-    headers = {'X-Auth-Token': token}
+    valid_token_headers = {'X-Auth-Token': token}
+    invalid_token_headers = {'X-Auth-Token': bad_token}
+
     m.get(requests_mock.ANY, status_code=404)
-    m.get(all_solver_url, status_code=403, request_headers={})
-    m.get(solver1_url, status_code=403, request_headers={})
-    m.get(solver2_url, status_code=403, request_headers={})
-    m.get(all_solver_url, text=two_solver_response, request_headers=headers)
-    m.get(solver1_url, text=first_solver_response, request_headers=headers)
-    m.get(solver2_url, text=second_solver_response, request_headers=headers)
+    m.get(requests_mock.ANY, status_code=401, request_headers=invalid_token_headers)
+
+    m.get(solver1_url, json=solver1_data, request_headers=valid_token_headers)
+    m.get(solver2_url, json=solver2_data, request_headers=valid_token_headers)
+    m.get(all_solver_url, json=all_solver_data, request_headers=valid_token_headers)
 
 
 class MockConnectivityTests(unittest.TestCase):
@@ -96,17 +93,19 @@ class MockConnectivityTests(unittest.TestCase):
 
     def test_bad_url(self):
         """Connect with a bad URL."""
-        with requests_mock.mock() as m:
+        with requests_mock.Mocker() as m:
             setup_server(m)
-            with self.assertRaises(IOError):
+            with self.assertRaises(SAPIError) as err:
                 with Client(bad_url, token) as client:
                     client.get_solvers()
+            # TODO: fix when exceptions/sapi call generalized
+            self.assertEqual(err.exception.error_code, 404)
 
     def test_bad_token(self):
         """Connect with a bad token."""
-        with requests_mock.mock() as m:
+        with requests_mock.Mocker() as m:
             setup_server(m)
-            with self.assertRaises(IOError):
+            with self.assertRaises(SolverAuthenticationError) as err:
                 with Client(url, bad_token) as client:
                     client.get_solvers()
 
@@ -141,16 +140,16 @@ class MockSolverLoading(unittest.TestCase):
             # test default, cached solver get
             with Client(url, token) as client:
                 # fetch solver not present in cache
-                solver = client.get_solver(solver_name)
-                self.assertEqual(solver.id, solver_name)
+                solver = client.get_solver(solver1_name)
+                self.assertEqual(solver.id, solver1_name)
 
                 # modify cached solver and re-fetch it
                 solver.id = 'different-solver'
                 # cached solver name doesn't match, so it won't be returned
                 with self.assertRaises(SolverError):
-                    client.get_solver(solver_name, refresh=False)
+                    client.get_solver(solver1_name, refresh=False)
                 # cache is refreshed?
-                self.assertEqual(client.get_solver(solver_name, refresh=True).id, solver_name)
+                self.assertEqual(client.get_solver(solver1_name, refresh=True).id, solver1_name)
 
     def test_load_all_solvers(self):
         """Load the list of solver names."""
@@ -189,24 +188,24 @@ class MockSolverLoading(unittest.TestCase):
             m.get(requests_mock.ANY, status_code=404)
             with Client(url, token) as client:
                 with self.assertRaises(SolverNotFoundError):
-                    client.get_solver(solver_name)
+                    client.get_solver(solver1_name)
 
     def test_load_solver_missing_data(self):
         """Try to load a solver that has incomplete data."""
         with requests_mock.mock() as m:
-            m.get(solver1_url, text=structured_solver_data(solver_name, incomplete=True))
+            m.get(solver1_url, json=structured_solver_data(solver1_name, incomplete=True))
             with Client(url, token) as client:
                 with self.assertRaises(SolverNotFoundError):
-                    client.get_solver(solver_name)
+                    client.get_solver(solver1_name)
 
     def test_load_solver_broken_response(self):
         """Try to load a solver for which the server has returned a truncated response."""
         with requests_mock.mock() as m:
-            body = structured_solver_data(solver_name)
+            body = json.dumps(structured_solver_data(solver1_name))
             m.get(solver1_url, text=body[0:len(body)//2])
             with Client(url, token) as client:
-                with self.assertRaises(ValueError):
-                    client.get_solver(solver_name)
+                with self.assertRaises(InvalidAPIResponseError):
+                    client.get_solver(solver1_name)
 
     def test_get_solver_reproducible(self):
         """get_solver should return same solver (assuming cache hasn't changed)"""
@@ -219,10 +218,10 @@ class MockSolverLoading(unittest.TestCase):
 
             with Client(url, token, defaults=defaults) as client:
                 solver = client.get_solver()
-                self.assertEqual(solver.id, second_solver_name)
+                self.assertEqual(solver.id, solver2_name)
 
                 solver = client.get_solver()
-                self.assertEqual(solver.id, second_solver_name)
+                self.assertEqual(solver.id, solver2_name)
 
     def test_solver_filtering_in_client(self):
         # base client
@@ -267,14 +266,14 @@ class MockSolverLoading(unittest.TestCase):
         self.assertFalse(solver_object('solver').has_flux_biases)
 
         # test .num_qubits vs .num_actual_qubits
-        data = json.loads(structured_solver_data('test'))
+        data = structured_solver_data('test')
         data['properties']['num_qubits'] = 7
         solver = Solver(None, data)
         self.assertEqual(solver.num_qubits, 7)
         self.assertEqual(solver.num_active_qubits, 3)
 
         # test .is_vfyc
-        data = json.loads(structured_solver_data('test'))
+        data = structured_solver_data('test')
         data['properties']['vfyc'] = 'error'
         self.assertFalse(Solver(None, data).is_vfyc)
         data['properties']['vfyc'] = True
@@ -292,7 +291,7 @@ class MockSolverLoading(unittest.TestCase):
 
         # test `.online` property
         self.assertTrue(solver_object('solver').online)
-        data = json.loads(structured_solver_data('test'))
+        data = structured_solver_data('test')
         data['status'] = 'offline'
         self.assertFalse(Solver(None, data).online)
         del data['status']
@@ -317,64 +316,3 @@ class MockSolverLoading(unittest.TestCase):
         self.assertFalse(solver_object('hybrid_x', cat='').qpu)
         self.assertFalse(solver_object('hybrid_x', cat='').software)
         self.assertTrue(solver_object('hybrid_x', cat='').hybrid)
-
-
-class RequestEvent(Exception):
-    """Throws exception when mocked client submits an HTTP request."""
-
-    def __init__(self, method, url, *args, **kwargs):
-        """Return the URL of the request with the exception for test verification."""
-        self.method = method
-        self.url = url
-        self.args = args
-        self.kwargs = kwargs
-
-    @staticmethod
-    def request(session, method, url, *args, **kwargs):
-        """Callback function that can be inserted into a mock."""
-        raise RequestEvent(method, url, *args, **kwargs)
-
-
-config_body = """
-[prod]
-endpoint = http://file-prod.url
-token = file-prod-token
-
-[alpha]
-endpoint = http://file-alpha.url
-token = file-alpha-token
-solver = alpha-solver
-
-[custom]
-endpoint = http://httpbin.org/delay/10
-token = 123
-permissive_ssl = True
-request_timeout = 15
-polling_timeout = 180
-"""
-
-
-class MockConfiguration(unittest.TestCase):
-
-    def test_custom_options(self):
-        """Test custom options (request_timeout, polling_timeout, permissive_ssl) are propagated to Client."""
-        request_timeout = 15
-        polling_timeout = 180
-
-        with mock.patch("dwave.cloud.config.open", iterable_mock_open(config_body), create=True):
-            with Client.from_config('config_file', profile='custom') as client:
-                # check permissive_ssl and timeouts custom params passed-thru
-                self.assertFalse(client.session.verify)
-                self.assertEqual(client.request_timeout, request_timeout)
-                self.assertEqual(client.polling_timeout, polling_timeout)
-
-                # verify client uses those properly
-                def mock_send(*args, **kwargs):
-                    self.assertEqual(kwargs.get('timeout'), request_timeout)
-                    response = requests.Response()
-                    response.status_code = 200
-                    response._content = b'{}'
-                    return response
-
-                with mock.patch("requests.adapters.HTTPAdapter.send", mock_send):
-                    client.get_solvers()
diff --git a/tests/test_mock_submission.py b/tests/test_mock_submission.py
index b6f4aac..b1baf7f 100644
--- a/tests/test_mock_submission.py
+++ b/tests/test_mock_submission.py
@@ -39,7 +39,8 @@ from dwave.cloud.utils import evaluate_ising, generate_const_ising_problem
 from dwave.cloud.client import Client, Solver
 from dwave.cloud.computation import Future
 from dwave.cloud.exceptions import (
-    SolverFailureError, CanceledFutureError, SolverError)
+    SolverFailureError, CanceledFutureError, SolverError,
+    InvalidAPIResponseError)
 
 
 def test_problem(solver):
@@ -188,10 +189,21 @@ def choose_reply(path, replies, statuses=None, date=None):
         response.text = replies[path]
         response.json.side_effect = lambda: json.loads(replies[path])
         response.headers = CaseInsensitiveDict({'Date': date.isoformat()})
+
         def raise_for_status():
-            if not 200 <= response.status_code < 300:
+            if not 200 <= response.status_code < 400:
                 raise HTTPError(response.status_code)
         response.raise_for_status = raise_for_status
+
+        def ok():
+            try:
+                response.raise_for_status()
+            except HTTPError:
+                return False
+            return True
+        ok_property = mock.PropertyMock(side_effect=ok)
+        type(response).ok = ok_property
+
         return response
     else:
         raise NotImplementedError(path)
@@ -246,7 +258,7 @@ class MockSubmission(_QueryTest):
                 linear, quadratic = test_problem(solver)
                 results = solver.sample_ising(linear, quadratic)
 
-                with self.assertRaises(IOError):
+                with self.assertRaises(InvalidAPIResponseError):
                     results.samples
 
     def test_submit_ising_ok_reply(self):
diff --git a/tests/test_solver.py b/tests/test_solver.py
index f28abc3..4db7d80 100644
--- a/tests/test_solver.py
+++ b/tests/test_solver.py
@@ -258,8 +258,9 @@ class Submission(_QueryTest):
 
             # simple problem with a large energy gap
             # (ground state: [-1, -1] @ -2.0)
-            h = {0: 1, 4: 1}
-            J = {(0, 4): -1}
+            n1, n2 = next(iter(solver.edges))
+            h = {n1: 1, n2: 1}
+            J = {(n1, n2): -1}
             offset = 1.0
 
             bqm = dimod.BinaryQuadraticModel.from_ising(h, J, offset)
diff --git a/tests/test_upload.py b/tests/test_upload.py
index ccc4d6d..6f0414d 100644
--- a/tests/test_upload.py
+++ b/tests/test_upload.py
@@ -26,7 +26,7 @@ from requests.exceptions import HTTPError
 
 from dwave.cloud.utils import tictoc
 from dwave.cloud.client import Client
-from dwave.cloud.exceptions import ProblemUploadError
+from dwave.cloud.exceptions import SAPIError, ProblemUploadError
 from dwave.cloud.upload import (
     Gettable, GettableFile, GettableMemory, FileView, ChunkedData)
 
@@ -419,10 +419,21 @@ def choose_reply(key, replies, statuses=None):
         response.status_code = next(statuses[key])
         response.text = replies[key]
         response.json.side_effect = lambda: json.loads(replies[key])
+
         def raise_for_status():
-            if not 200 <= response.status_code < 300:
+            if not 200 <= response.status_code < 400:
                 raise HTTPError(response.status_code)
         response.raise_for_status = raise_for_status
+
+        def ok():
+            try:
+                response.raise_for_status()
+            except HTTPError:
+                return False
+            return True
+        ok_property = mock.PropertyMock(side_effect=ok)
+        type(response).ok = ok_property
+
         return response
     else:
         raise NotImplementedError(key)
@@ -804,7 +815,7 @@ class TestMultipartUpload(unittest.TestCase):
 
         with Client(**config) as client:
             with client.create_session() as session:
-                with self.assertRaisesRegex(ProblemUploadError,
+                with self.assertRaisesRegex(SAPIError,
                                             'bigger than the maximum'):
                     client._initiate_multipart_upload(session, size)
 
diff --git a/tests/test_utils.py b/tests/test_utils.py
index a1a3dc5..5212a65 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -22,11 +22,13 @@ from collections import OrderedDict
 from itertools import count
 from datetime import datetime
 
+from parameterized import parameterized
+
 from dwave.cloud.utils import (
     uniform_iterator, uniform_get, strip_head, strip_tail,
     active_qubits, generate_random_ising_problem,
     default_text_input, utcnow, cached, retried, deprecated, aliasdict,
-    parse_loglevel, user_agent)
+    parse_loglevel, user_agent, hasinstance, exception_chain, is_caused_by)
 
 
 class TestSimpleUtils(unittest.TestCase):
@@ -549,5 +551,92 @@ class TestAliasdict(unittest.TestCase):
         self.assertDictEqual(new.aliases, aliases)
 
 
+class TestExceptionUtils(unittest.TestCase):
+
+    def raise_implicit():
+        try:
+            1/0
+        except:
+            raise ValueError
+
+    def raise_explicit():
+        try:
+            1/0
+        except Exception as e:
+            raise ValueError from e
+
+    def raise_mixed():
+        try:
+            TestExceptionUtils.raise_explicit()
+        except:
+            raise TypeError
+
+    def test_hasinstance(self):
+        # not contained
+        self.assertFalse(hasinstance([], ValueError))
+        self.assertFalse(hasinstance([TypeError], ValueError))
+        self.assertFalse(hasinstance([TypeError()], ValueError))
+        self.assertFalse(hasinstance([ValueError], ValueError))
+
+        # contained in unit list
+        self.assertTrue(hasinstance([ValueError()], ValueError))
+        self.assertTrue(hasinstance([ValueError('msg')], ValueError))
+
+        # contained in a list
+        self.assertTrue(hasinstance([TypeError(), ValueError()], ValueError))
+        self.assertTrue(hasinstance([ValueError(), ValueError()], ValueError))
+
+        # contained in a tuple
+        self.assertTrue(hasinstance((TypeError(), ValueError()), ValueError))
+
+        # base class also contained
+        self.assertTrue(hasinstance([ValueError()], Exception))
+
+        # multiple types can be checked
+        self.assertFalse(hasinstance([ValueError()], (dict, list)))
+        self.assertTrue(hasinstance([ValueError()], (dict, Exception)))
+        self.assertTrue(hasinstance([ValueError()], (ValueError, TypeError)))
+        self.assertTrue(hasinstance([TypeError()], (ValueError, TypeError)))
+        self.assertTrue(hasinstance([TypeError(), ValueError], (ValueError, TypeError)))
+
+    @parameterized.expand([
+        (raise_implicit, (ValueError, ZeroDivisionError)),
+        (raise_explicit, (ValueError, ZeroDivisionError)),
+        (raise_mixed, (TypeError, ValueError, ZeroDivisionError))
+    ])
+    def test_exception_chain(self, raise_exc, chained_types):
+        try:
+            raise_exc()
+        except Exception as e:
+            exc = e
+        else:
+            self.fail('Exception not raised')
+
+        chain = list(exception_chain(exc))
+        self.assertEqual(len(chain), len(chained_types))
+
+        for idx, typ in enumerate(chained_types):
+            self.assertIsInstance(chain[idx], typ)
+
+        for typ in chained_types:
+            self.assertTrue(hasinstance(exception_chain(exc), typ))
+
+    @parameterized.expand([
+        (raise_implicit, (ValueError, ZeroDivisionError)),
+        (raise_explicit, (ValueError, ZeroDivisionError)),
+        (raise_mixed, (TypeError, ValueError, ZeroDivisionError))
+    ])
+    def test_is_caused_by(self, raise_exc, exception_types):
+        try:
+            raise_exc()
+        except Exception as exc:
+            self.assertTrue(is_caused_by(exc, exception_types))
+
+            for typ in exception_types:
+                self.assertTrue(is_caused_by(exc, typ))
+
+            self.assertFalse(is_caused_by(exc, KeyError))
+
+
 if __name__ == '__main__':
     unittest.main()