Repository: dwavesystems/dwave-cloud-client

Issue:
Assume 5xx responses during problem polling are transient errors, don't abort polling
Although this can be incorrect, as a good first approximation we can assume server-side errors (5xx) are temporary, meaning we should not stop polling, but rather retry later.

At the same time we're assuming client-side errors (4xx) are unrecoverable by retrying.

Discussion:

Patch:
diff --git a/dwave/cloud/client.py b/dwave/cloud/client.py
index ff37d7e..886eb16 100644
--- a/dwave/cloud/client.py
+++ b/dwave/cloud/client.py
@@ -1030,18 +1030,22 @@ class Client(object):
     def _handle_problem_status(self, message, future):
         """Handle the results of a problem submission or results request.
 
-        This method checks the status of the problem and puts it in the correct queue.
+        This method checks the status of the problem and puts it in the correct
+        queue.
 
         Args:
-            message (dict): Update message from the SAPI server wrt. this problem.
-            future `Future`: future corresponding to the problem
+            message (dict):
+                Update message from the SAPI server wrt. this problem.
+            future (:class:`dwave.cloud.computation.Future`:
+                future corresponding to the problem
 
         Note:
             This method is always run inside of a daemon thread.
         """
         try:
             logger.trace("Handling response: %r", message)
-            logger.debug("Handling response for %s with status %s", message.get('id'), message.get('status'))
+            logger.debug("Handling response for %s with status %s",
+                         message.get('id'), message.get('status'))
 
             # Handle errors in batch mode
             if 'error_code' in message and 'error_msg' in message:
@@ -1273,6 +1277,7 @@ class Client(object):
                 else:
                     logger.trace("Skipping non-positive delay of %.2f sec", delay)
 
+                # execute and handle the polling request
                 try:
                     logger.trace("Executing poll API request")
 
@@ -1283,11 +1288,27 @@ class Client(object):
 
                     if response.status_code == 401:
                         raise SolverAuthenticationError()
-                    response.raise_for_status()
 
-                    statuses = response.json()
-                    for status in statuses:
-                        self._handle_problem_status(status, frame_futures[status['id']])
+                    # assume 5xx errors are transient, and don't abort polling
+                    if 500 <= response.status_code < 600:
+                        logger.warning(
+                            "Received an internal server error response on "
+                            "problem status polling request (%s). Assuming "
+                            "error is transient, and resuming polling.",
+                            response.status_code)
+                        # add all futures in this frame back to the polling queue
+                        # XXX: logic split between `_handle_problem_status` and here
+                        for future in frame_futures.values():
+                            self._poll(future)
+
+                    else:
+                        # otherwise, fail
+                        response.raise_for_status()
+
+                        # or handle a successful request
+                        statuses = response.json()
+                        for status in statuses:
+                            self._handle_problem_status(status, frame_futures[status['id']])
 
                 except BaseException as exception:
                     if not isinstance(exception, SolverAuthenticationError):


Test patch:
diff --git a/tests/test_mock_submission.py b/tests/test_mock_submission.py
index eb7ae73..1b6ecb2 100644
--- a/tests/test_mock_submission.py
+++ b/tests/test_mock_submission.py
@@ -21,11 +21,13 @@ import json
 import unittest
 import itertools
 import threading
+import collections
 
 from datetime import datetime, timedelta
 from dateutil.tz import UTC
 from dateutil.parser import parse as parse_datetime
 from requests.structures import CaseInsensitiveDict
+from requests.exceptions import HTTPError
 
 from dwave.cloud.utils import evaluate_ising, generate_random_ising_problem
 from dwave.cloud.client import Client, Solver
@@ -149,17 +151,24 @@ def continue_reply(id_, solver_name, now=None, eta_min=None, eta_max=None):
     return json.dumps(resp)
 
 
-def choose_reply(path, replies, date=None):
+def choose_reply(path, replies, statuses=None, date=None):
     """Choose the right response based on the path and make a mock response."""
 
+    if statuses is None:
+        statuses = collections.defaultdict(lambda: iter([200]))
+
     if date is None:
         date = datetime_in_future(0)
 
     if path in replies:
         response = mock.Mock(['json', 'raise_for_status', 'headers'])
-        response.status_code = 200
+        response.status_code = next(statuses[path])
         response.json.side_effect = lambda: json.loads(replies[path])
         response.headers = CaseInsensitiveDict({'Date': date.isoformat()})
+        def raise_for_status():
+            if not 200 <= response.status_code < 300:
+                raise HTTPError(response.status_code)
+        response.raise_for_status = raise_for_status
         return response
     else:
         raise NotImplementedError(path)
@@ -475,6 +484,47 @@ class MockSubmission(_QueryTest):
                 future = solver.sample_qubo({})
                 future.result()
 
+    # Reduce the number of poll and submission threads so that the system can be tested
+    @mock.patch.object(Client, "_POLL_THREAD_COUNT", 1)
+    @mock.patch.object(Client, "_SUBMISSION_THREAD_COUNT", 1)
+    def test_polling_recovery_after_5xx(self):
+        "Polling shouldn't be aborted on 5xx responses."
+
+        with Client('endpoint', 'token') as client:
+            client.session = mock.Mock()
+            # on submit, return status pending
+            client.session.post = lambda path, _: choose_reply(path, {
+                'endpoint/problems/': '[%s]' % continue_reply('123', 'abc123')
+            })
+            # on first and second status poll, fail with 503 and 504
+            # on third status poll, return completed
+            statuses = iter([503, 504])
+            def continue_then_complete(path, state={'count': 0}):
+                state['count'] += 1
+                if state['count'] < 3:
+                    return choose_reply(path, replies={
+                        'endpoint/problems/?id=123': '[%s]' % continue_reply('123', 'abc123'),
+                        'endpoint/problems/123/': continue_reply('123', 'abc123')
+                    }, statuses={
+                        'endpoint/problems/?id=123': statuses,
+                        'endpoint/problems/123/': statuses
+                    })
+                else:
+                    return choose_reply(path, {
+                        'endpoint/problems/?id=123': '[%s]' % complete_no_answer_reply('123', 'abc123'),
+                        'endpoint/problems/123/': complete_reply('123', 'abc123')
+                    })
+
+            client.session.get = continue_then_complete
+
+            solver = Solver(client, solver_data('abc123'))
+
+            future = solver.sample_qubo({})
+            future.result()
+
+            # after third poll, back-off interval should be 4 x initial back-off
+            self.assertEqual(future._poll_backoff, Client._POLL_BACKOFF_MIN * 2**2)
+
 
 class DeleteEvent(Exception):
     """Throws exception when mocked client submits an HTTP DELETE request."""