Repository: dwavesystems/dwave-cloud-client

Issue:
Add support for uploading compressed QPU problem data
Add a config parameter to enable compression during upload of QPU problem data (default to ~off~ **on**).

Discussion:
codecov[bot]: ## [Codecov](https://app.codecov.io/gh/dwavesystems/dwave-cloud-client/pull/655?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dwavesystems) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 88.90%. Comparing base [(`ed81913`)](https://app.codecov.io/gh/dwavesystems/dwave-cloud-client/commit/ed81913ec792eb768f02366e003b845129f45877?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dwavesystems) to head [(`98dd49d`)](https://app.codecov.io/gh/dwavesystems/dwave-cloud-client/commit/98dd49d07a11cd0dc96fcf16f3a8f29051503780?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dwavesystems).

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master     #655      +/-   ##
==========================================
+ Coverage   88.84%   88.90%   +0.05%     
==========================================
  Files          42       42              
  Lines        4789     4796       +7     
==========================================
+ Hits         4255     4264       +9     
+ Misses        534      532       -2     
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/dwavesystems/dwave-cloud-client/pull/655?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dwavesystems).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dwavesystems).


Patch:
diff --git a/dwave/cloud/client/base.py b/dwave/cloud/client/base.py
index e0a9a38..9355004 100644
--- a/dwave/cloud/client/base.py
+++ b/dwave/cloud/client/base.py
@@ -49,6 +49,7 @@ import base64
 import hashlib
 import codecs
 import concurrent.futures
+import zlib
 
 from itertools import chain, zip_longest
 from functools import partial, wraps
@@ -158,6 +159,10 @@ class Client(object):
             Force HTTP(S) connection close after each request. Set to ``True``
             to prevent intermediate network equipment closing idle connections.
 
+        compress_qpu_problem_data (bool, default=True):
+            Enable QPU problem data compression on upload to SAPI. Enabled by
+            default. Set to ``False`` to disable compression.
+
         headers (dict/str, optional):
             Newline-separated additional HTTP headers to include with each
             API request, or a dictionary of (key, value) pairs.
@@ -305,6 +310,7 @@ class Client(object):
         'request_timeout': 60,
         'polling_timeout': None,
         'connection_close': False,
+        'compress_qpu_problem_data': True,
         'headers': None,
         'client_cert': None,
         'client_cert_key': None,
@@ -1285,11 +1291,19 @@ class Client(object):
                 # Submit the problems
                 logger.debug("Submitting %d problems", len(ready_problems))
                 try:
-                    body = orjson.dumps([
+                    data = orjson.dumps([
                         orjson.Fragment(msg.body.result()) for msg in ready_problems
                     ])
-                    logger.debug('Size of POST body = %d', len(body))
-                    message = Client._sapi_request(session.post, 'problems/', data=body)
+                    logger.debug('Size of problems/jobs data = %d', len(data))
+
+                    headers = {}
+                    if self.config.compress_qpu_problem_data:
+                        data = zlib.compress(data)
+                        headers['Content-Encoding'] = 'deflate'
+                        logger.debug("Compressed with 'deflate', new size = %d", len(data))
+
+                    message = Client._sapi_request(
+                        session.post, 'problems/', data=data, headers=headers)
                     logger.debug("Finished submitting %d problems", len(ready_problems))
 
                 except Exception as exc:
diff --git a/dwave/cloud/config/models.py b/dwave/cloud/config/models.py
index cc62303..cb1c3fb 100644
--- a/dwave/cloud/config/models.py
+++ b/dwave/cloud/config/models.py
@@ -163,6 +163,7 @@ class ClientConfig(BaseModel, GetterMixin):
     # specific connection options
     permissive_ssl: Optional[bool] = False
     connection_close: Optional[bool] = False
+    compress_qpu_problem_data: Optional[bool] = True
 
     # api request retry params
     request_retry: Optional[RequestRetryConfig] = RequestRetryConfig()
@@ -312,6 +313,7 @@ _V1_CONFIG_DEFAULTS = {
     'request_timeout': 60,
     'polling_timeout': None,
     'connection_close': False,
+    'compress_qpu_problem_data': True,
     'headers': None,
     'client_cert': None,
     'client_cert_key': None,
diff --git a/releasenotes/notes/add-compress-qpu-problem-data-option-86e708d10f80477a.yaml b/releasenotes/notes/add-compress-qpu-problem-data-option-86e708d10f80477a.yaml
new file mode 100644
index 0000000..8c8ee67
--- /dev/null
+++ b/releasenotes/notes/add-compress-qpu-problem-data-option-86e708d10f80477a.yaml
@@ -0,0 +1,9 @@
+---
+features:
+  - |
+    Enable QPU problem data compression on upload using ``deflate`` content
+    encoding.
+
+    Compression is controlled with a new config option, ``compress_qpu_problem_data``,
+    defaulting to true, and it can be turned off either in config file or via
+    ``Client()`` kwarg.


Test patch:
diff --git a/tests/test_mock_submission.py b/tests/test_mock_submission.py
index 1a271d5..b317296 100644
--- a/tests/test_mock_submission.py
+++ b/tests/test_mock_submission.py
@@ -17,6 +17,7 @@
 import threading
 import time
 import unittest
+import zlib
 from concurrent.futures import TimeoutError, ThreadPoolExecutor
 from typing import Any, Dict
 from unittest import mock
@@ -84,9 +85,7 @@ class _QueryTest:
             self.assertEqual(state, list(record.sample))
 
 
-@mock.patch('time.sleep', lambda *x: None)
-class MockSubmissionBaseTests(_QueryTest):
-    """Test connecting and some related failure modes."""
+class MockSubmissionBase(_QueryTest):
 
     @classmethod
     def setUpClass(cls):
@@ -94,8 +93,14 @@ class MockSubmissionBaseTests(_QueryTest):
         cls.config = dict(
             endpoint='endpoint',
             token='token',
-            poll_strategy=cls.poll_strategy,
         )
+        if hasattr(cls, 'poll_strategy'):
+            cls.config['poll_strategy'] = cls.poll_strategy
+
+
+@mock.patch('time.sleep', lambda *x: None)
+class MockSubmissionBaseTests(MockSubmissionBase):
+    """Test connecting and some related failure modes."""
 
     def test_submit_null_reply(self):
         """Get an error when the server's response is incomplete."""
@@ -424,7 +429,12 @@ class MockSubmissionBaseTests(_QueryTest):
                                              self.sapi.error_reply(id='1')]
                     }, **self.poll_params))
 
-            def accept_problems_with_continue_reply(path, data, ids=iter('12')):
+            def accept_problems_with_continue_reply(path, data=None, headers=None, ids=iter('12')):
+                if headers is None:
+                    headers = {}
+                encoding = headers.get('Content-Encoding', 'identity').lower()
+                if encoding == 'deflate':
+                    data = zlib.decompress(data)
                 problems = orjson.loads(data)
                 return choose_reply(path, {
                     'problems/': [self.sapi.continue_reply(id=next(ids)) for _ in problems]
@@ -452,6 +462,60 @@ class MockSubmissionBaseTests(_QueryTest):
                 self._check(results2, linear, quadratic, **params)
 
 
+class TestUploadCompression(MockSubmissionBase, unittest.TestCase):
+    """Verify `compress_qpu_problem_data` config directive is respected, i.e.
+    QPU problem data is compressed on upload, and Content-Encoding is correctly
+    set."""
+
+    @parameterized.expand([
+        (True, ),
+        (False, ),
+    ])
+    def test_compression_on_upload(self, compress):
+        # make sure POST data is compressed if `compress_qpu_problem_data` config
+        # option is set
+
+        class Invalid(Exception):
+            pass
+
+        def create_mock_session(client):
+            session = mock.Mock()
+
+            def post(path, **kwargs):
+                data = kwargs.pop('data')
+                headers = kwargs.pop('headers', {})
+                encoding = headers.get('Content-Encoding', 'identity')
+
+                if compress and encoding == 'deflate':
+                    data = zlib.decompress(data)
+
+                if compress and encoding != 'deflate':
+                    raise Invalid
+
+                problem = orjson.loads(data)
+                if problem[0]['data'] != self.sapi.problem_data():
+                    raise Invalid
+
+                return choose_reply(path, {
+                    'problems/': [self.sapi.complete_no_answer_reply(id='123')]})
+
+            session.post = post
+            session.get = lambda path, **kwargs: choose_reply(path, {
+                'problems/123/': self.sapi.complete_reply(id='123')})
+
+            return session
+
+        with mock.patch.object(Client, 'create_session', create_mock_session):
+            with Client(compress_qpu_problem_data=compress, **self.config) as client:
+                solver = Solver(client, self.sapi.solver.data)
+
+                linear, quadratic = self.sapi.problem
+                params = dict(num_reads=100)
+                results = solver.sample_ising(linear, quadratic, **params)
+
+                self._check(results, linear, quadratic, **params)
+
+
 class MockSubmissionWithShortPolling(MockSubmissionBaseTests,
                                      unittest.TestCase):