Repository: dwavesystems/dwave-cloud-client
Pull request #435 (2020-10-08 21:51:07)

Accept looser file-like for upload
We now use `DQM.to_file()`, instead of relying on the internal `_file` attribute.

Obsoletes https://github.com/dwavesystems/dimod/issues/730.


Patch:
diff --git a/dwave/cloud/solver.py b/dwave/cloud/solver.py
index 561dd93..1335a7c 100644
--- a/dwave/cloud/solver.py
+++ b/dwave/cloud/solver.py
@@ -525,10 +525,7 @@ class DQMSolver(BaseUnstructuredSolver):
 
     def _encode_problem_for_upload(self, dqm):
         try:
-            # note: SpooledTemporaryFile currently returned by DQM.to_file
-            # does not implement io.BaseIO interface, so we use the underlying
-            # (and internal) file-like object for now
-            data = dqm.to_file()._file
+            data = dqm.to_file()
         except Exception as e:
             logger.debug("DQM conversion to file failed with %r, "
                          "assuming data already encoded.", e)
@@ -544,6 +541,7 @@ class DQMSolver(BaseUnstructuredSolver):
         # to sample BQM problems, we need to convert them to DQM
         dqm = bqm_to_dqm(bqm)
 
+        # TODO: convert sampleset back
         return self.sample_dqm(dqm, **params)
 
     def sample_dqm(self, dqm, **params):
diff --git a/dwave/cloud/upload.py b/dwave/cloud/upload.py
index 87bc306..00794ec 100644
--- a/dwave/cloud/upload.py
+++ b/dwave/cloud/upload.py
@@ -153,7 +153,8 @@ class GettableFile(GettableBase):
                 isinstance(f, (io.BufferedIOBase, io.RawIOBase))
                 and f.seekable() and f.readable())
         else:
-            valid = lambda f: all([hasattr(f, 'readinto'), hasattr(f, 'seek')])
+            valid = lambda f: all([
+                hasattr(f, 'readinto'), hasattr(f, 'seek'), hasattr(f, 'tell')])
 
         if not valid(fp):
             raise TypeError("expected file-like, seekable, readable object")
@@ -161,8 +162,9 @@ class GettableFile(GettableBase):
         # store file size, assuming it won't change
         self._size = fp.seek(0, os.SEEK_END)
         if self._size is None:
-            # handle python2, when `fp.seek()` is `file.seek()`
-            # note: not a thread-safe way
+            # handle non-python3 and/or non-standard file seek() impl.
+            # (like tempfile.SpooledTemporaryFile)
+            # note: not thread-safe!
             self._size = fp.tell()
 
         self._fp = fp
@@ -340,16 +342,9 @@ class ChunkedData(object):
 
     """
 
-    def __init__(self, data, chunk_size):
-        self.data = data
-        self.view = None
-        self.chunk_size = int(chunk_size)
-
-        if self.chunk_size <= 0:
-            raise ValueError("positive integer required for chunk size")
-
+    def _thread_safe_data_view(self, data):
         # convenience string handler
-        if isinstance(data, str) and not isinstance(data, bytes):
+        if isinstance(data, str):
             data = data.encode('ascii')
 
         if isinstance(data, (bytes, bytearray)):
@@ -357,20 +352,32 @@ class ChunkedData(object):
             # buffer) only for `bytes` objects, and make a copy otherwise!
 
             # use non-locking memory view over data buffer
-            self.view = FileView(GettableMemory(data))
+            return FileView(GettableMemory(data))
 
-        if self.view is None and isinstance(data, io.IOBase):
-            # use locking file view if possible
-            if not data.seekable():
-                raise ValueError("seekable file-like data object expected")
-            if not data.readable():
-                raise ValueError("readable file-like data object expected")
-            self.view = FileView(GettableFile(data))
+        # use locking file view if possible
+        try:
+            return FileView(GettableFile(data, strict=True))
+        except TypeError:
+            logger.debug("data does not conform to strict file-like requirements")
 
         # TODO: use stream view if possible
 
-        if self.view is None:
-            raise TypeError("bytes/str/IOBase-subclass data required")
+        # fallback to a less strict check of file-like's capabilities,
+        # accepting we might fail later
+        try:
+            return FileView(GettableFile(data, strict=False))
+        except TypeError:
+            logger.debug("data does not conform to loose file-like requirements")
+
+        raise TypeError("bytes/str/file-like data required")
+
+    def __init__(self, data, chunk_size):
+        self.data = data
+        self.view = self._thread_safe_data_view(data)
+
+        if chunk_size <= 0:
+            raise ValueError("positive integer required for chunk size")
+        self.chunk_size = int(chunk_size)
 
     @property
     def total_size(self):
diff --git a/setup.py b/setup.py
index c159292..867f3bc 100644
--- a/setup.py
+++ b/setup.py
@@ -27,7 +27,7 @@ extras_require = {
     'bqm': ['dimod>=0.8.15', 'numpy>=1.16'],
 
     # dqm support
-    'bqm': ['dimod>=0.9.7', 'numpy>=1.16'],
+    'dqm': ['dimod>=0.9.7', 'numpy>=1.16'],
 }
 
 # Packages provided. Only include packages under the 'dwave' namespace.


Test patch:
diff --git a/tests/test_mock_unstructured_submission.py b/tests/test_mock_unstructured_submission.py
index a49f719..087d28f 100644
--- a/tests/test_mock_unstructured_submission.py
+++ b/tests/test_mock_unstructured_submission.py
@@ -83,64 +83,70 @@ class TestUnstructuredSolver(unittest.TestCase):
             return Present(result=mock_problem_id)
 
         # construct a functional solver by mocking client and api response data
-        with mock.patch.object(Client, 'create_session', lambda self: session):
+        with mock.patch.multiple(Client, create_session=lambda self: session,
+                                 upload_problem_encoded=mock_upload):
             with Client('endpoint', 'token') as client:
-                with mock.patch.object(BaseUnstructuredSolver, 'upload_problem', mock_upload):
-                    solver = BQMSolver(client, unstructured_solver_data())
-
-                    # make sure this still works
-                    _ = UnstructuredSolver(client, unstructured_solver_data())
-
-                    # direct bqm sampling
-                    ss = dimod.ExactSolver().sample(bqm)
-                    session.post = lambda path, _: choose_reply(
-                        path, {'problems/': complete_reply(ss)})
-
-                    fut = solver.sample_bqm(bqm)
-                    numpy.testing.assert_array_equal(fut.sampleset, ss)
-                    numpy.testing.assert_array_equal(fut.samples, ss.record.sample)
-                    numpy.testing.assert_array_equal(fut.energies, ss.record.energy)
-                    numpy.testing.assert_array_equal(fut.num_occurrences, ss.record.num_occurrences)
-
-                    # submit of pre-uploaded bqm problem
-                    fut = solver.sample_bqm(mock_problem_id)
-                    numpy.testing.assert_array_equal(fut.sampleset, ss)
-                    numpy.testing.assert_array_equal(fut.samples, ss.record.sample)
-                    numpy.testing.assert_array_equal(fut.energies, ss.record.energy)
-                    numpy.testing.assert_array_equal(fut.num_occurrences, ss.record.num_occurrences)
-
-                    # ising sampling
-                    lin, quad, _ = bqm.to_ising()
-                    ss = dimod.ExactSolver().sample_ising(lin, quad)
-                    session.post = lambda path, _: choose_reply(
-                        path, {'problems/': complete_reply(ss)})
-
-                    fut = solver.sample_ising(lin, quad)
-                    numpy.testing.assert_array_equal(fut.sampleset, ss)
-                    numpy.testing.assert_array_equal(fut.samples, ss.record.sample)
-                    numpy.testing.assert_array_equal(fut.energies, ss.record.energy)
-                    numpy.testing.assert_array_equal(fut.num_occurrences, ss.record.num_occurrences)
-
-                    # qubo sampling
-                    qubo, _ = bqm.to_qubo()
-                    ss = dimod.ExactSolver().sample_qubo(qubo)
-                    session.post = lambda path, _: choose_reply(
-                        path, {'problems/': complete_reply(ss)})
-
-                    fut = solver.sample_qubo(qubo)
-                    numpy.testing.assert_array_equal(fut.sampleset, ss)
-                    numpy.testing.assert_array_equal(fut.samples, ss.record.sample)
-                    numpy.testing.assert_array_equal(fut.energies, ss.record.energy)
-                    numpy.testing.assert_array_equal(fut.num_occurrences, ss.record.num_occurrences)
+                solver = BQMSolver(client, unstructured_solver_data())
+
+                # make sure this still works
+                _ = UnstructuredSolver(client, unstructured_solver_data())
+
+                # direct bqm sampling
+                ss = dimod.ExactSolver().sample(bqm)
+                session.post = lambda path, _: choose_reply(
+                    path, {'problems/': complete_reply(ss)})
+
+                fut = solver.sample_bqm(bqm)
+                numpy.testing.assert_array_equal(fut.sampleset, ss)
+                numpy.testing.assert_array_equal(fut.samples, ss.record.sample)
+                numpy.testing.assert_array_equal(fut.energies, ss.record.energy)
+                numpy.testing.assert_array_equal(fut.num_occurrences, ss.record.num_occurrences)
+
+                # submit of pre-uploaded bqm problem
+                fut = solver.sample_bqm(mock_problem_id)
+                numpy.testing.assert_array_equal(fut.sampleset, ss)
+                numpy.testing.assert_array_equal(fut.samples, ss.record.sample)
+                numpy.testing.assert_array_equal(fut.energies, ss.record.energy)
+                numpy.testing.assert_array_equal(fut.num_occurrences, ss.record.num_occurrences)
+
+                # ising sampling
+                lin, quad, _ = bqm.to_ising()
+                ss = dimod.ExactSolver().sample_ising(lin, quad)
+                session.post = lambda path, _: choose_reply(
+                    path, {'problems/': complete_reply(ss)})
+
+                fut = solver.sample_ising(lin, quad)
+                numpy.testing.assert_array_equal(fut.sampleset, ss)
+                numpy.testing.assert_array_equal(fut.samples, ss.record.sample)
+                numpy.testing.assert_array_equal(fut.energies, ss.record.energy)
+                numpy.testing.assert_array_equal(fut.num_occurrences, ss.record.num_occurrences)
+
+                # qubo sampling
+                qubo, _ = bqm.to_qubo()
+                ss = dimod.ExactSolver().sample_qubo(qubo)
+                session.post = lambda path, _: choose_reply(
+                    path, {'problems/': complete_reply(ss)})
+
+                fut = solver.sample_qubo(qubo)
+                numpy.testing.assert_array_equal(fut.sampleset, ss)
+                numpy.testing.assert_array_equal(fut.samples, ss.record.sample)
+                numpy.testing.assert_array_equal(fut.energies, ss.record.energy)
+                numpy.testing.assert_array_equal(fut.num_occurrences, ss.record.num_occurrences)
 
     def test_sample_dqm_smoke_test(self):
         """Construction of and sampling from an unstructured DQM solver works."""
-        # note: full test pending dimod release that supports DQM
 
-        # mock a DQM problem
-        #bqm = dimod.BQM.from_ising({}, {'ab': 1})
-        dqm = mock.Mock()
-        dqm.to_file.return_value._file = io.BytesIO(b'123')
+        try:
+            import dimod
+            dqm = dimod.DQM()
+            dqm.add_variable(5)
+            dqm.add_variable(7)
+            dqm.set_linear_case(0, 3, 1.5)
+            dqm.set_quadratic(0, 1, {(0, 1): 1.5, (3, 4): 1})
+        except:
+            # dimod or dimod with DQM support not available, so just use a mock
+            dqm = mock.Mock()
+            dqm.to_file.return_value = io.BytesIO(b'123')
 
         problem_type = 'dqm'
 
@@ -153,20 +159,20 @@ class TestUnstructuredSolver(unittest.TestCase):
             return Present(result=mock_problem_id)
 
         # construct a functional solver by mocking client and api response data
-        with mock.patch.object(Client, 'create_session', lambda self: session):
+        with mock.patch.multiple(Client, create_session=lambda self: session,
+                                 upload_problem_encoded=mock_upload):
             with Client('endpoint', 'token') as client:
-                with mock.patch.object(BaseUnstructuredSolver, 'upload_problem', mock_upload):
-                    solver = DQMSolver(client, unstructured_solver_data(problem_type=problem_type))
+                solver = DQMSolver(client, unstructured_solver_data(problem_type=problem_type))
 
-                    # use bqm for now
-                    ss = dimod.ExactSolver().sample(dimod.BQM.empty('SPIN'))
-                    session.post = lambda path, _: choose_reply(
-                        path, {'problems/': complete_reply(ss, problem_type=problem_type)})
+                # use bqm for mock response (for now)
+                ss = dimod.ExactSolver().sample(dimod.BQM.empty('SPIN'))
+                session.post = lambda path, _: choose_reply(
+                    path, {'problems/': complete_reply(ss, problem_type=problem_type)})
 
-                    # verify decoding works
-                    fut = solver.sample_dqm(dqm)
-                    numpy.testing.assert_array_equal(fut.sampleset, ss)
-                    numpy.testing.assert_array_equal(fut.problem_type, problem_type)
+                # verify decoding works
+                fut = solver.sample_dqm(dqm)
+                numpy.testing.assert_array_equal(fut.sampleset, ss)
+                numpy.testing.assert_array_equal(fut.problem_type, problem_type)
 
     def test_upload_failure(self):
         """Submit should gracefully fail if upload as part of submit fails."""
diff --git a/tests/test_upload.py b/tests/test_upload.py
index 4a62f68..ccc4d6d 100644
--- a/tests/test_upload.py
+++ b/tests/test_upload.py
@@ -347,6 +347,46 @@ class TestChunkedData(unittest.TestCase):
         chunks_expected = [b'012', b'345', b'678', b'9']
         self.verify_chunking(cd, chunks_expected)
 
+    def test_chunks_from_bqm(self):
+        try:
+            from dimod import AdjVectorBQM
+            AdjVectorBQM.to_file
+        except (ImportError, AttributeError):
+            self.skipTest("dimod.AdjVectorBQM with .to_file() unavailable")
+
+        # serialize a BQM via .to_file
+        bqm = AdjVectorBQM.from_ising({'a': 1}, {})
+        bqmfile = bqm.to_file()     # returns dimod's FileView
+
+        chunk_size = 10
+        cd = ChunkedData(bqmfile, chunk_size=chunk_size)
+
+        # verify chunks
+        bqmfile.seek(0)
+        raw = bqmfile.read()
+        chunks_expected = [raw[i:i+chunk_size] for i in range(0, len(raw), chunk_size)]
+        self.verify_chunking(cd, chunks_expected)
+
+    def test_chunks_from_dqm(self):
+        try:
+            from dimod import DQM
+        except ImportError:
+            self.skipTest("dimod.DQM unavailable")
+
+        # serialize a DQM via .to_file
+        dqm = DQM()
+        dqm.add_variable(1)
+        dqmfile = dqm.to_file()    # returns SpooledTemporaryFile subclass
+
+        chunk_size = 100
+        cd = ChunkedData(dqmfile, chunk_size=chunk_size)
+
+        # verify chunks
+        dqmfile.seek(0)
+        raw = dqmfile.read()
+        chunks_expected = [raw[i:i+chunk_size] for i in range(0, len(raw), chunk_size)]
+        self.verify_chunking(cd, chunks_expected)
+
     def test_chunk_size_edges(self):
         with self.assertRaises(ValueError):
             cd = ChunkedData(self.data, chunk_size=0)