Repository: mars-project/mars

Issue:
[BUG] Run the example code hangs in distributed mode
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**
Create a Mars cluster and run the code in readme:
``` Python
import mars.tensor as mt
N = 200_000_000
a = mt.random.uniform(-1, 1, size=(N, 2))
print(((mt.linalg.norm(a, axis=1) < 1)
        .sum() * 4 / N).execute())
```

it hangs and error be found in server client:
```
2020-11-09 21:30:22,053 mars.scheduler.operands.common 97 ERROR    Attempt 1: Unexpected error KeyError occurred in executing operand 230bef1901408a5f9134f34444918898 in 11.238.146.2:35131
Traceback (most recent call last):
  File "/home/admin/work/public-mars-0.5.4.zip/mars/promise.py", line 378, in _wrapped
    return func(*args, **kwargs)
  File "/home/admin/work/public-mars-0.5.4.zip/mars/utils.py", line 365, in _wrapped
    return func(*args, **kwargs)
  File "/home/admin/work/public-mars-0.5.4.zip/mars/worker/execution.py", line 564, in execute_graph
    quota_request = self._prepare_quota_request(session_id, graph_key)
  File "/home/admin/work/public-mars-0.5.4.zip/mars/worker/execution.py", line 249, in _prepare_quota_request
    memory_estimations = self._estimate_calc_memory(session_id, graph_key)
  File "/home/admin/work/public-mars-0.5.4.zip/mars/worker/execution.py", line 213, in _estimate_calc_memory
    res = executor.execute_graph(graph_record.graph, graph_record.chunk_targets, mock=True)
  File "/home/admin/work/public-mars-0.5.4.zip/mars/executor.py", line 690, in execute_graph
    res = graph_execution.execute(retval)
  File "/home/admin/work/public-mars-0.5.4.zip/mars/executor.py", line 571, in execute
    future.result()
  File "/home/admin/work/public-mars-0.5.4.zip/mars/executor.py", line 186, in result
    raise self._exc_info[1] from None
  File "/home/admin/work/public-mars-0.5.4.zip/mars/executor.py", line 198, in submit
    return self._MockResult(fn(*args, **kwargs))
  File "/home/admin/work/public-mars-0.5.4.zip/mars/utils.py", line 439, in _inner
    return func(*args, **kwargs)
  File "/home/admin/work/public-mars-0.5.4.zip/mars/executor.py", line 443, in _execute_operand
    Executor.handle(first_op, results, self._mock)
  File "/home/admin/work/public-mars-0.5.4.zip/mars/executor.py", line 641, in handle
    return runner(results, op)
  File "/home/admin/work/public-mars-0.5.4.zip/mars/tensor/fuse/ne.py", line 75, in estimate_size
    estimate_fuse_size(ctx, op)
  File "/home/admin/work/public-mars-0.5.4.zip/mars/tensor/fuse/core.py", line 49, in estimate_fuse_size
    results = executor.execute_graph(dag, output_keys, mock=True, no_intermediate=True)
  File "/home/admin/work/public-mars-0.5.4.zip/mars/executor.py", line 690, in execute_graph
    res = graph_execution.execute(retval)
  File "/home/admin/work/public-mars-0.5.4.zip/mars/executor.py", line 571, in execute
    future.result()
  File "/opt/conda/lib/python3.7/concurrent/futures/_base.py", line 428, in result
    return self.__get_result()
  File "/opt/conda/lib/python3.7/concurrent/futures/_base.py", line 384, in __get_result
    raise self._exception
  File "/opt/conda/lib/python3.7/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/home/admin/work/public-mars-0.5.4.zip/mars/utils.py", line 439, in _inner
    return func(*args, **kwargs)
  File "/home/admin/work/public-mars-0.5.4.zip/mars/executor.py", line 486, in _execute_operand
    del results[dep_key]
KeyError: '94e11781368129674925eb2d4ae093bf'
```



Discussion:

Resolving patch:
diff --git a/mars/tensor/fuse/core.py b/mars/tensor/fuse/core.py
--- a/mars/tensor/fuse/core.py
+++ b/mars/tensor/fuse/core.py
@@ -30,6 +30,7 @@ def __init__(self, dtype=None, **kw):
 def estimate_fuse_size(ctx, op):
     from ...graph import DAG
     from ...executor import Executor
+    from ...utils import build_fetch_chunk
 
     chunk = op.outputs[0]
     dag = DAG()
@@ -40,6 +41,7 @@ def estimate_fuse_size(ctx, op):
         for inp in c.inputs:
             if inp.key not in keys:
                 size_ctx[inp.key] = ctx[inp.key]
+                inp = build_fetch_chunk(inp).data
             if inp not in dag:
                 dag.add_node(inp)
             dag.add_edge(inp, c)


Test patch:
diff --git a/mars/deploy/local/tests/test_cluster.py b/mars/deploy/local/tests/test_cluster.py
--- a/mars/deploy/local/tests/test_cluster.py
+++ b/mars/deploy/local/tests/test_cluster.py
@@ -261,11 +261,13 @@ def testSingleOutputTensorExecute(self, *_):
             self.assertTrue(np.isscalar(res))
             self.assertLess(res, 10)
 
-            t = mt.random.rand(10)
-            r = t.sum() * 4 - 1
+            raw = np.random.rand(10)
+            t = mt.tensor(raw)
+            r = (mt.linalg.norm(t) * 4 - 1).sum()
 
             res = r.to_numpy()
-            self.assertLess(res, 39)
+            expected = (np.linalg.norm(raw) * 4 - 1).sum()
+            np.testing.assert_array_almost_equal(res, expected)
 
     def testMultipleOutputTensorExecute(self, *_):
         with new_cluster(scheduler_n_process=2, worker_n_process=2,