Repository: mars-project/mars

Issue:
[BUG] Estimated size is inaccurate for fuse operand
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**
For fuse operand, median result size is larger than output which leads to the inaccurate size estimate, we should use the maximum size as estimated value.

**To Reproduce**
```
In [60]: a = mt.random.rand(10, 10, chunk_size=10) 
    ...: b = a[:, mt.newaxis, :] - a 
    ...: r = mt.triu(mt.sqrt(b ** 2).sum(axis=2))                                                                           

In [61]: size_executor.execute_tensor(r, mock=True)                                                                         
Out[61]: [(800, 800)]
```
When print the size context, the maximum size is 8000.
```
{'57be8a87cf4b925bf77ab7e98b5368fc': (800, 800)}
{'57be8a87cf4b925bf77ab7e98b5368fc': (800, 800), '519f3ee39cfbe2d6466d6dcb74e1e660': (800, 800)}
{'57be8a87cf4b925bf77ab7e98b5368fc': (800, 800), '519f3ee39cfbe2d6466d6dcb74e1e660': (800, 800), '8e837df3951221d2942096496e20617a': (8000, 8000)}
{'8e837df3951221d2942096496e20617a': (8000, 8000), '982381b14ad27aa7da48ee6b5a7d52ef': (8000, 8000)}
{'57be8a87cf4b925bf77ab7e98b5368fc': (800, 800), '519f3ee39cfbe2d6466d6dcb74e1e660': (800, 800), '982381b14ad27aa7da48ee6b5a7d52ef': (8000, 8000)}
{'982381b14ad27aa7da48ee6b5a7d52ef': (8000, 8000), '957220bcc23420c1addcadc1264310d6': (8000, 8000)}
{'57be8a87cf4b925bf77ab7e98b5368fc': (800, 800), '519f3ee39cfbe2d6466d6dcb74e1e660': (800, 800), '957220bcc23420c1addcadc1264310d6': (8000, 8000)}
{'957220bcc23420c1addcadc1264310d6': (8000, 8000), 'e3aae97df3f54c91cdf748a8b8d88949': (800, 8000)}
{'57be8a87cf4b925bf77ab7e98b5368fc': (800, 800), '519f3ee39cfbe2d6466d6dcb74e1e660': (800, 800), 'e3aae97df3f54c91cdf748a8b8d88949': (800, 8000)}
{'e3aae97df3f54c91cdf748a8b8d88949': (800, 8000), 'f1b128ece76c1cd9504753eb5f1442a4': (800, 800)}

```




Discussion:

Resolving patch:
diff --git a/mars/executor.py b/mars/executor.py
--- a/mars/executor.py
+++ b/mars/executor.py
@@ -175,7 +175,7 @@ class GraphExecution(object):
     """
     def __init__(self, chunk_results, graph, keys, executed_keys, sync_provider,
                  n_parallel=None, prefetch=False, print_progress=False,
-                 mock=False, sparse_mock_percent=1.0):
+                 mock=False, mock_max_memory=0):
         self._chunk_results = chunk_results
         self._graph = graph
         self._keys = keys
@@ -184,7 +184,7 @@ def __init__(self, chunk_results, graph, keys, executed_keys, sync_provider,
         self._prefetch = prefetch
         self._print_progress = print_progress
         self._mock = mock
-        self._sparse_mock_percent = sparse_mock_percent
+        self._mock_max_memory = mock_max_memory
 
         # pool executor for the operand execution
         self._operand_executor = sync_provider.thread_pool_executor(self._n_parallel)
@@ -260,6 +260,14 @@ def _execute_operand(self, op):
             # so we pass the first operand's first output to Executor.handle
             first_op = ops[0]
             Executor.handle(first_op.outputs[0], results, self._mock)
+
+            # update maximal memory usage during execution
+            if self._mock:
+                self._mock_max_memory = max(
+                    self._mock_max_memory,
+                    sum(results[op_output.key][1] for op_output in first_op.outputs
+                        if results.get(op_output.key) is not None))
+
             executed_chunk_keys.update([c.key for c in first_op.outputs])
             op_keys.add(first_op.key)
             # handle other operands
@@ -370,6 +378,13 @@ def execute(self, retval=True):
         for future in executed_futures:
             future.result()
 
+        # update with the maximal memory cost during the whole execution
+        if self._mock:
+            avg_max_mem = self._mock_max_memory // len(self._keys)
+            for key in self._keys:
+                r = self._chunk_results[key]
+                self._chunk_results[key] = (r[0], max(r[1], avg_max_mem))
+
         if retval:
             return [self._chunk_results[key] for key in self._keys]
 
@@ -405,6 +420,8 @@ def __init__(self, engine=None, storage=None, prefetch=False,
         # synchronous provider
         self._sync_provider = self._sync_provider[sync_provider_type]
 
+        self._mock_max_memory = 0
+
     @property
     def chunk_result(self):
         return self._chunk_result
@@ -437,7 +454,7 @@ def handle(cls, chunk, results, mock=False):
             return cls._get_op_runner(chunk, cls._op_size_estimators)(results, chunk)
 
     def execute_graph(self, graph, keys, n_parallel=None, print_progress=False,
-                      mock=False, sparse_mock_percent=1.0):
+                      mock=False):
         optimized_graph = self._preprocess(graph, keys)
 
         executed_keys = list(itertools.chain(*[v[1] for v in self.stored_tensors.values()]))
@@ -445,15 +462,16 @@ def execute_graph(self, graph, keys, n_parallel=None, print_progress=False,
                                          keys, executed_keys, self._sync_provider,
                                          n_parallel=n_parallel, prefetch=self._prefetch,
                                          print_progress=print_progress, mock=mock,
-                                         sparse_mock_percent=sparse_mock_percent)
+                                         mock_max_memory=self._mock_max_memory)
         res = graph_execution.execute(True)
+        self._mock_max_memory = max(self._mock_max_memory, graph_execution._mock_max_memory)
         if mock:
             self._chunk_result.clear()
         return res
 
     @kernel_mode
     def execute_tensor(self, tensor, n_parallel=None, n_thread=None, concat=False,
-                       print_progress=False, mock=False, sparse_mock_percent=1.0):
+                       print_progress=False, mock=False):
         if concat:
             # only for tests
             tensor.tiles()
@@ -469,12 +487,11 @@ def execute_tensor(self, tensor, n_parallel=None, n_thread=None, concat=False,
 
         return self.execute_graph(graph, [c.key for c in tensor.chunks],
                                   n_parallel=n_parallel or n_thread,
-                                  print_progress=print_progress, mock=mock,
-                                  sparse_mock_percent=sparse_mock_percent)
+                                  print_progress=print_progress, mock=mock)
 
     @kernel_mode
     def execute_tensors(self, tensors, fetch=True, n_parallel=None, n_thread=None,
-                        print_progress=False, mock=False, compose=True, sparse_mock_percent=1.0):
+                        print_progress=False, mock=False, compose=True):
         graph = DirectedGraph()
 
         result_keys = []
@@ -512,8 +529,7 @@ def execute_tensors(self, tensors, fetch=True, n_parallel=None, n_thread=None,
                                executed_keys=list(self._chunk_result.keys()))
 
         self.execute_graph(graph, result_keys, n_parallel=n_parallel or n_thread,
-                           print_progress=print_progress, mock=mock,
-                           sparse_mock_percent=sparse_mock_percent)
+                           print_progress=print_progress, mock=mock)
 
         results = self._chunk_result
         try:


Test patch:
diff --git a/mars/tests/test_executor.py b/mars/tests/test_executor.py
--- a/mars/tests/test_executor.py
+++ b/mars/tests/test_executor.py
@@ -81,3 +81,14 @@ def testActorInExecutor(self):
         with create_actor_pool(n_process=2) as pool:
             actor = pool.create_actor(ExecutorActor, uid='0-executor')
             self.assertIsNone(actor.send(1))
+
+    def testMockExecuteSize(self):
+        a = mt.random.rand(10, 10, chunk_size=10)
+        b = a[:, mt.newaxis, :] - a
+        r = mt.triu(mt.sqrt(b ** 2).sum(axis=2))
+
+        executor = Executor()
+        res = executor.execute_tensor(r, concat=False, mock=True)
+        # larger than maximal memory size in calc procedure
+        self.assertGreaterEqual(res[0][0], 800)
+        self.assertGreaterEqual(res[0][1], 8000)