Repository: mars-project/mars

Issue:
[BUG] Execution failed and `InvalidComposedNodeError` was raised
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**
`InvalidComposedNodeError` was raised when executing tensors.  In the below case, a fused chunk is fused into another fuse chunk which leads to the error.

**To Reproduce**
``` Python
In [1]: import mars.tensor as mt                                                                                                    

In [2]: a = mt.random.rand(4, 4) + 1 - 3                                                                                            

In [3]: u, s, v = mt.linalg.svd(a)                                                                                                  

In [4]: (u + 1).execute()                                                                                                           
Out[4]: 
array([[0.53491564, 0.50844154, 1.31934217, 0.33660917],
       [0.48043745, 0.58359388, 1.12894514, 1.73486995],
       [0.54248881, 1.05235106, 0.12344775, 0.86000348],
       [0.4482439 , 1.76304364, 1.33622891, 0.98327479]])

In [5]: (u + 2).execute()                                                                                                           
---------------------------------------------------------------------------
InvalidComposedNodeError                  Traceback (most recent call last)
<ipython-input-5-1dd2365be547> in <module>
----> 1 (u + 2).execute()

~/Documents/mars_dev/mars/mars/tensor/core.py in execute(self, session, **kw)
    442 
    443     def execute(self, session=None, **kw):
--> 444         return self._data.execute(session, **kw)
    445 
    446 

~/Documents/mars_dev/mars/mars/core.py in execute(self, session, **kw)
    572         if session is None:
    573             session = Session.default_or_local()
--> 574         return session.run(self, **kw)
    575 
    576     def fetch(self, session=None, **kw):

~/Documents/mars_dev/mars/mars/session.py in run(self, *tileables, **kw)
    150         tileables = tuple(mt.tensor(t) if not isinstance(t, (Entity, Base)) else t
    151                           for t in tileables)
--> 152         result = self._sess.run(*tileables, **kw)
    153 
    154         for t in tileables:

~/Documents/mars_dev/mars/mars/session.py in run(self, *tileables, **kw)
     61         if 'n_parallel' not in kw:
     62             kw['n_parallel'] = cpu_count()
---> 63         res = self._executor.execute_tileables(tileables, **kw)
     64         return res
     65 

~/Documents/mars_dev/mars/mars/utils.py in _wrapped(*args, **kwargs)
    389                 _kernel_mode.eager = False
    390             _kernel_mode.eager_count = enter_eager_count + 1
--> 391             return func(*args, **kwargs)
    392         finally:
    393             _kernel_mode.eager_count -= 1

~/Documents/mars_dev/mars/mars/executor.py in execute_tileables(self, tileables, fetch, n_parallel, n_thread, print_progress, mock, compose)
    548 
    549         self.execute_graph(graph, result_keys, n_parallel=n_parallel or n_thread,
--> 550                            print_progress=print_progress, mock=mock)
    551 
    552         results = self._chunk_result

~/Documents/mars_dev/mars/mars/executor.py in execute_graph(self, graph, keys, n_parallel, print_progress, mock, no_intermediate, compose, retval)
    469         :return: execution result
    470         """
--> 471         optimized_graph = self._preprocess(graph, keys) if compose else graph
    472 
    473         if not mock:

~/Documents/mars_dev/mars/mars/executor.py in _preprocess(self, graph, keys)
    435         from .optimizes.core import Optimizer
    436 
--> 437         Optimizer(graph, self._engine).optimize(keys=keys)
    438         return graph
    439 

~/Documents/mars_dev/mars/mars/optimizes/core.py in optimize(self, keys)
     32 
     33     def optimize(self, keys=None):
---> 34         self._graph.decompose()
     35         if self._engine == 'numpy':
     36             return

~/Documents/mars_dev/mars/mars/graph.pyx in mars.graph.DirectedGraph.decompose()

~/Documents/mars_dev/mars/mars/fuse.pyx in mars.fuse.Fusion.decompose()

~/Documents/mars_dev/mars/mars/fuse.pyx in mars.fuse.Fusion._decompose_node()

InvalidComposedNodeError: Invalid composed node data


```


Discussion:

Resolving patch:
diff --git a/mars/executor.py b/mars/executor.py
--- a/mars/executor.py
+++ b/mars/executor.py
@@ -543,8 +543,13 @@ def execute_tileables(self, tileables, fetch=True, n_parallel=None, n_thread=Non
             else:
                 concat_keys.append(tileable.chunks[0].key)
 
-            tileable.build_graph(graph=graph, tiled=True, compose=compose,
+            # Do not do compose here, because building graph has not finished yet
+            tileable.build_graph(graph=graph, tiled=True, compose=False,
                                  executed_keys=list(self._chunk_result.keys()))
+        if compose:
+            # finally do compose according to option
+            graph.compose(keys=list(itertools.chain(*[[c.key for c in t.chunks]
+                                                      for t in tileables])))
 
         self.execute_graph(graph, result_keys, n_parallel=n_parallel or n_thread,
                            print_progress=print_progress, mock=mock)
diff --git a/mars/tensor/linalg/matmul.py b/mars/tensor/linalg/matmul.py
--- a/mars/tensor/linalg/matmul.py
+++ b/mars/tensor/linalg/matmul.py
@@ -183,7 +183,12 @@ def execute(cls, ctx, op):
                 # tell sparse to do calculation on numpy or cupy matmul
                 ctx[op.outputs[0].key] = xp.matmul(a, b, sparse=False)
             else:
-                ctx[op.outputs[0].key] = xp.matmul(a, b, casting=op.casting, order=op.order)
+                try:
+                    # `np.matmul` support `order` argument in version 1.16
+                    ctx[op.outputs[0].key] = xp.matmul(a, b, casting=op.casting, order=op.order)
+                except TypeError:  # pragma: no cover
+                    ctx[op.outputs[0].key] = xp.matmul(a, b).astype(dtype=op.dtype,
+                                                                    casting=op.casting, order=op.order)
 
 
 def matmul(a, b, sparse=None, out=None, **kw):


Test patch:
diff --git a/mars/optimizes/tests/test_compose.py b/mars/optimizes/tests/test_compose.py
--- a/mars/optimizes/tests/test_compose.py
+++ b/mars/optimizes/tests/test_compose.py
@@ -83,14 +83,14 @@ def testCompose(self):
         self.assertIn(composed_nodes[0], graph.successors(chunks[0]))
         self.assertIn(composed_nodes[0], graph.successors(chunks[1]))
         # check composed's inputs
-        self.assertIn(chunks[0].data, composed_nodes[0].inputs)
-        self.assertIn(chunks[1].data, composed_nodes[0].inputs)
+        self.assertIn(chunks[0].key, [n.key for n in composed_nodes[0].inputs])
+        self.assertIn(chunks[1].key, [n.key for n in composed_nodes[0].inputs])
         # check composed's predecessors
         self.assertIn(chunks[0], graph.predecessors(composed_nodes[0]))
         self.assertIn(chunks[1], graph.predecessors(composed_nodes[0]))
         # check 4 and 5's inputs
-        self.assertIn(composed_nodes[0].data, graph.successors(composed_nodes[0])[0].inputs)
-        self.assertIn(composed_nodes[0].data, graph.successors(composed_nodes[0])[0].inputs)
+        self.assertIn(composed_nodes[0].key, [n.key for n in graph.successors(composed_nodes[0])[0].inputs])
+        self.assertIn(composed_nodes[0].key, [n.key for n in graph.successors(composed_nodes[0])[0].inputs])
         # check 4 and 5's predecessors
         self.assertIn(composed_nodes[0], graph.predecessors(chunks[4]))
         self.assertIn(composed_nodes[0], graph.predecessors(chunks[5]))
diff --git a/mars/tests/test_session.py b/mars/tests/test_session.py
--- a/mars/tests/test_session.py
+++ b/mars/tests/test_session.py
@@ -63,10 +63,15 @@ def testMultipleOutputExecute(self):
 
         # test multiple outputs, but only execute 1
         data = np.random.randint(0, 10, (5, 5))
-        arr3 = mt.tensor(data)
+        arr3 = (mt.tensor(data) + 1) * 2
         arrs = mt.linalg.qr(arr3)
-        result = arrs[0].execute()
-        expected = np.linalg.qr(data)[0]
+        result = (arrs[0] + 1).execute()
+        expected = np.linalg.qr((data + 1) * 2)[0] + 1
+
+        np.testing.assert_array_almost_equal(result, expected)
+
+        result = (arrs[0] + 2).execute()
+        expected = np.linalg.qr((data + 1) * 2)[0] + 2
 
         np.testing.assert_array_almost_equal(result, expected)
 
@@ -337,7 +342,7 @@ def testDataFrameTensorConvert(self):
         # test exception
         tensor = md.from_tensor(mt.array([1, 2, 3]))
         np.testing.assert_equal(sess.run(tensor), np.array([1, 2, 3]).reshape(3, 1))
-        
+
     def testFetchSlices(self):
         sess = new_session()