Repository: mars-project/mars

Issue:
[BUG] Fail to execute SVD of matrix of ones
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**To Reproduce**
``` Python
In [1]: import mars.tensor as mt                                                                                                   

In [2]: a = mt.ones((10, 5), chunk_size=5)                                                                                         

In [3]: s = mt.linalg.svd(a)[1]                                                                                                    

In [4]: s.execute()                                                                                                                
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-4-4a5966ced510> in <module>
----> 1 s.execute()

~/Documents/mars_dev/mars/mars/tensor/core.py in execute(self, session, **kw)
    378 
    379     def execute(self, session=None, **kw):
--> 380         return self._data.execute(session, **kw)
    381 
    382 

~/Documents/mars_dev/mars/mars/tensor/core.py in execute(self, session, **kw)
    184         if session is None:
    185             session = Session.default_or_local()
--> 186         return session.run(self, **kw)
    187 
    188     def fetch(self, session=None, **kw):

~/Documents/mars_dev/mars/mars/session.py in run(self, *tensors, **kw)
    107 
    108         tensors = tuple(mt.tensor(t) for t in tensors)
--> 109         result = self._sess.run(*tensors, **kw)
    110 
    111         for t in tensors:

~/Documents/mars_dev/mars/mars/session.py in run(self, *tensors, **kw)
     49         if 'n_parallel' not in kw:
     50             kw['n_parallel'] = cpu_count()
---> 51         res = self._executor.execute_tensors(tensors, **kw)
     52         return res
     53 

~/Documents/mars_dev/mars/mars/utils.py in _wrapped(*args, **kwargs)
    343     def _wrapped(*args, **kwargs):
    344         _kernel_mode.eager = False
--> 345         return_value = func(*args, **kwargs)
    346         _kernel_mode.eager = None
    347         return return_value

~/Documents/mars_dev/mars/mars/executor.py in execute_tensors(self, tensors, fetch, n_parallel, n_thread, print_progress, mock, sparse_mock_percent)
    473         self.execute_graph(graph, result_keys, n_parallel=n_parallel or n_thread,
    474                            print_progress=print_progress, mock=mock,
--> 475                            sparse_mock_percent=sparse_mock_percent)
    476 
    477         results = self._chunk_result

~/Documents/mars_dev/mars/mars/executor.py in execute_graph(self, graph, keys, n_parallel, print_progress, mock, sparse_mock_percent)
    410                                          print_progress=print_progress, mock=mock,
    411                                          sparse_mock_percent=sparse_mock_percent)
--> 412         return graph_execution.execute(True)
    413 
    414     @kernel_mode

~/Documents/mars_dev/mars/mars/executor.py in execute(self, retval)
    339         # wait until all the futures completed
    340         for future in executed_futures:
--> 341             future.result()
    342 
    343         if self._mock:

~/miniconda3/lib/python3.7/concurrent/futures/_base.py in result(self, timeout)
    423                 raise CancelledError()
    424             elif self._state == FINISHED:
--> 425                 return self.__get_result()
    426 
    427             self._condition.wait(timeout)

~/miniconda3/lib/python3.7/concurrent/futures/_base.py in __get_result(self)
    382     def __get_result(self):
    383         if self._exception:
--> 384             raise self._exception
    385         else:
    386             return self._result

~/miniconda3/lib/python3.7/concurrent/futures/thread.py in run(self)
     55 
     56         try:
---> 57             result = self.fn(*self.args, **self.kwargs)
     58         except BaseException as exc:
     59             self.future.set_exception(exc)

~/Documents/mars_dev/mars/mars/executor.py in _execute_operand(self, op)
    246                     # and some of the output not in result keys, delete them
    247                     if ref_counts.get(output.key) == 0:
--> 248                         del results[output.key]
    249 
    250                     # clean the predecessors' results if ref counts equals 0

KeyError: '4ec5b76155f38976c84e36d156349503'
```

**Additional context**
The issue may be related to #216 .


Discussion:
This is also relative to the refactoring of Executor in #212 , and this could only happen on 0.2.x branch.

Resolving patch:
diff --git a/mars/_version.py b/mars/_version.py
--- a/mars/_version.py
+++ b/mars/_version.py
@@ -16,7 +16,7 @@
 import os
 import sys
 
-version_info = (0, 2, 0, 'a1')
+version_info = (0, 2, 0, 'a2')
 _num_index = max(idx if isinstance(v, int) else 0
                  for idx, v in enumerate(version_info))
 __version__ = '.'.join(map(str, version_info[:_num_index + 1])) + \
diff --git a/mars/executor.py b/mars/executor.py
--- a/mars/executor.py
+++ b/mars/executor.py
@@ -219,6 +219,8 @@ def _execute_operand(self, op):
         results = self._chunk_results
         ref_counts = self._chunk_key_ref_counts
         op_keys = self._executed_op_keys
+        executed_chunk_keys = set()
+        deleted_chunk_keys = set()
         try:
             ops = list(self._op_key_to_ops[op.key])
             if not self._mock:
@@ -227,11 +229,15 @@ def _execute_operand(self, op):
                 # so we pass the first operand's first output to Executor.handle
                 first_op = ops[0]
                 Executor.handle(first_op.outputs[0], results)
+                executed_chunk_keys.update([c.key for c in first_op.outputs])
                 op_keys.add(first_op.key)
                 # handle other operands
                 for rest_op in ops[1:]:
                     for op_output, rest_op_output in zip(first_op.outputs, rest_op.outputs):
-                        results[rest_op_output.key] = results[op_output.key]
+                        # if the op's outputs have been stored,
+                        # other same key ops' results will be the same
+                        if rest_op_output.key not in executed_chunk_keys:
+                            results[rest_op_output.key] = results[op_output.key]
             else:
                 sparse_percent = self._sparse_mock_percent if op.sparse else 1.0
                 for output in op.outputs:
@@ -245,7 +251,10 @@ def _execute_operand(self, op):
                     # in case that operand has multiple outputs
                     # and some of the output not in result keys, delete them
                     if ref_counts.get(output.key) == 0:
-                        del results[output.key]
+                        # if the result has been deleted, it should be skipped
+                        if output.key not in deleted_chunk_keys:
+                            deleted_chunk_keys.add(output.key)
+                            del results[output.key]
 
                     # clean the predecessors' results if ref counts equals 0
                     for pred_chunk in self._graph.iter_predecessors(output):


Test patch:
diff --git a/mars/tensor/execution/tests/test_linalg_execute.py b/mars/tensor/execution/tests/test_linalg_execute.py
--- a/mars/tensor/execution/tests/test_linalg_execute.py
+++ b/mars/tensor/execution/tests/test_linalg_execute.py
@@ -112,6 +112,15 @@ def testSVDExecution(self):
         res = self.executor.execute_tensor(t, concat=True)[0]
         self.assertTrue(np.allclose(res, data))
 
+        # test for matrix of ones
+        data = np.ones((20, 10))
+
+        a = tensor(data, chunk_size=10)
+        s = svd(a)[1]
+        res = self.executor.execute_tensor(s, concat=True)[0]
+        expected = np.linalg.svd(a)[1]
+        np.testing.assert_array_almost_equal(res, expected)
+
     def testCholeskyExecution(self):
         data = np.array([[1, -2j], [2j, 5]])