Repository: mars-project/mars

Issue:
[BUG] Chained remote functions failed
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**
Chained remote functions will fail due to fuse.

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version
2. The version of Mars you use
3. Versions of crucial packages, such as numpy, scipy and protobuf
4. Full stack of the error.
5. Minimized code to reproduce the error.

```
In [1]: import mars.remote as mr                                                

In [2]: def f(x): 
   ...:     return x + 1 
   ...:                                                                         

In [3]: def g(x): 
   ...:     return x * 2 
   ...:                                                                         

In [4]: s = mr.spawn(g, mr.spawn(f, 2))                                         

In [5]: s.execute()                                                             
---------------------------------------------------------------------------
NotImplementedError                       Traceback (most recent call last)
<ipython-input-5-4a5966ced510> in <module>
----> 1 s.execute()

~/Workspace/mars/mars/core.py in execute(self, session, **kw)
    368 
    369         # no more fetch, thus just fire run
--> 370         session.run(self, **kw)
    371         # return Tileable or ExecutableTuple itself
    372         return self

~/Workspace/mars/mars/session.py in run(self, *tileables, **kw)
    398         tileables = tuple(mt.tensor(t) if not isinstance(t, (Entity, Base)) else t
    399                           for t in tileables)
--> 400         result = self._sess.run(*tileables, **kw)
    401 
    402         for t in tileables:

~/Workspace/mars/mars/session.py in run(self, *tileables, **kw)
     97             # set number of running cores
     98             self.context.set_ncores(kw['n_parallel'])
---> 99             res = self._executor.execute_tileables(tileables, **kw)
    100             return res
    101 

~/Workspace/mars/mars/utils.py in _wrapped(*args, **kwargs)
    383                 _kernel_mode.eager = False
    384             _kernel_mode.eager_count = enter_eager_count + 1
--> 385             return func(*args, **kwargs)
    386         finally:
    387             _kernel_mode.eager_count -= 1

~/Workspace/mars/mars/utils.py in inner(*args, **kwargs)
    471     def inner(*args, **kwargs):
    472         with build_mode():
--> 473             return func(*args, **kwargs)
    474     return inner
    475 

~/Workspace/mars/mars/executor.py in execute_tileables(self, tileables, fetch, n_parallel, n_thread, print_progress, mock, compose)
    832                 # compose
    833                 if compose:
--> 834                     chunk_graph.compose(list(temp_result_keys))
    835                 # execute chunk graph
    836                 self.execute_graph(chunk_graph, list(temp_result_keys),

~/Workspace/mars/mars/graph.pyx in mars.graph.DirectedGraph.compose()

~/Workspace/mars/mars/optimizes/chunk_graph/fuse.pyx in mars.optimizes.chunk_graph.fuse.Fusion.compose()

~/Workspace/mars/mars/optimizes/chunk_graph/fuse.pyx in mars.optimizes.chunk_graph.fuse.Fusion._compose_graph()

~/Workspace/mars/mars/utils.py in build_fuse_chunk(fused_chunks, **kwargs)
    583     params = tail_chunk.params.copy()
    584 
--> 585     fuse_op = chunk_op.get_fuse_op_cls(tail_chunk)(
    586         sparse=chunk_op.sparse, _key=chunk_op.key, _gpu=tail_chunk.op.gpu,
    587         _operands=[c.op for c in fused_chunks])

~/Workspace/mars/mars/operands.py in get_fuse_op_cls(self, obj)
    433 
    434     def get_fuse_op_cls(self, obj):
--> 435         raise NotImplementedError
    436 
    437 

NotImplementedError: 
```



Discussion:

Resolving patch:
diff --git a/mars/remote/core.py b/mars/remote/core.py
--- a/mars/remote/core.py
+++ b/mars/remote/core.py
@@ -19,13 +19,14 @@
 from ..context import ContextBase
 from ..core import Entity, Base
 from ..serialize import FunctionField, ListField, DictField, BoolField, Int32Field
-from ..operands import ObjectOperand, ObjectOperandMixin
+from ..operands import ObjectOperand
 from ..tensor.core import TENSOR_TYPE
 from ..dataframe.core import DATAFRAME_TYPE, SERIES_TYPE, INDEX_TYPE
+from .operands import RemoteOperandMixin
 from .utils import replace_inputs, find_objects
 
 
-class RemoteFunction(ObjectOperand, ObjectOperandMixin):
+class RemoteFunction(ObjectOperand, RemoteOperandMixin):
     _op_type_ = opcodes.REMOTE_FUNCATION
     _op_module_ = 'remote'
 
diff --git a/mars/remote/operands.py b/mars/remote/operands.py
new file mode 100644
--- /dev/null
+++ b/mars/remote/operands.py
@@ -0,0 +1,34 @@
+# Copyright 1999-2020 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ..core import FuseChunkData, FuseChunk
+from ..operands import Fuse, FuseChunkMixin, ObjectOperandMixin
+
+
+class RemoteFuseChunkMixin(FuseChunkMixin, ObjectOperandMixin):
+    __slots__ = ()
+
+    def _create_chunk(self, output_idx, index, **kw):
+        data = FuseChunkData(_index=index, _op=self, **kw)
+
+        return FuseChunk(data)
+
+
+class RemoteFuseChunk(Fuse, RemoteFuseChunkMixin):
+    pass
+
+
+class RemoteOperandMixin(ObjectOperandMixin):
+    def get_fuse_op_cls(self, _):
+        return RemoteFuseChunk


Test patch:
diff --git a/mars/deploy/local/tests/test_cluster.py b/mars/deploy/local/tests/test_cluster.py
--- a/mars/deploy/local/tests/test_cluster.py
+++ b/mars/deploy/local/tests/test_cluster.py
@@ -1009,6 +1009,11 @@ def g(x, y):
             r = session.run(c, timeout=_exec_timeout)
             self.assertEqual(r, 20)
 
+            e = mr.spawn(f, mr.spawn(f, 2))
+
+            r = session.run(e, timeout=_exec_timeout)
+            self.assertEqual(r, 4)
+
             session2 = new_session(cluster.endpoint)
             expect_session_id = session2.session_id
 
diff --git a/mars/remote/tests/test_remote_function.py b/mars/remote/tests/test_remote_function.py
--- a/mars/remote/tests/test_remote_function.py
+++ b/mars/remote/tests/test_remote_function.py
@@ -102,3 +102,15 @@ def reducer(word_to_count_list):
             result.update(wc)
 
         self.assertEqual(result, {'word1': 2, 'word2': 3, 'word3': 2})
+
+    def testChainedRemote(self):
+        def f(x):
+            return x + 1
+
+        def g(x):
+            return x * 2
+
+        s = spawn(g, spawn(f, 2))
+
+        result = self.executor.execute_tileables([s])[0]
+        self.assertEqual(result, 6)