Repository: mars-project/mars

Issue:
[BUG] Duplicate operands execution
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**

The case is `mars.core.entity.tests.test_utils.test_recursive_tile`. It generates a chunk graph with duplicate operands (the op key are same, the execution output are same).

I added some code to check the operands in stage.py:

``` python
class TaskStageProcessor:
    def __init__(
        self,
        stage_id: str,
        task: Task,
        chunk_graph: ChunkGraph,
        subtask_graph: SubtaskGraph,
        bands: List[BandType],
        tile_context: Dict[TileableType, TileableType],
        scheduling_api: SchedulingAPI,
        meta_api: MetaAPI,
    ):
        self.stage_id = stage_id
        self.task = task
        self.chunk_graph = chunk_graph
        self.subtask_graph = subtask_graph
        self._bands = bands
        self._tile_context = tile_context

        # Check for duplicate operand.
        from collections import Counter
        unique_keys = Counter(c.key for c in chunk_graph)
        if len(unique_keys) < len(chunk_graph):
            import pprint
            pprint.pprint([c for c in chunk_graph])
            pprint.pprint([k for k, v in unique_keys.items() if v > 1])
            subtask_graph.view()
```

The output is:

```python
[Chunk <op=TensorRand, key=560bdcf1ae4978a999cbe3fd620c65d7_0>,
 Chunk <op=TensorRand, key=5aed4a149f5084ce95b9212afb61f24d_0>,
 Chunk <op=TensorRand, key=6fa461793a5d0feef9905c945653653f_0>,
 Chunk <op=TensorRand, key=688bd5f2d8fe39ce3e1a8af91b53b65c_0>,
 Chunk <op=TensorSlice, key=a9cd6ffd308ed11c3fccd2753f9a4aba_0>,
 Chunk <op=PSRSShuffle, stage=reduce, key=94c1f8fa627bcd5960ad795f7c692671_0>,
 Chunk <op=TensorShuffleProxy, key=6e7794b968faabcbebdfb289789c1ab9_0>,
 Chunk <op=PSRSShuffle, stage=map, key=0ebf73717ef2479456a6351c6209a90e_0>,
 Chunk <op=PSRSShuffle, stage=map, key=eb053dc8499bc7cdc12d8382f199e096_0>,
 Chunk <op=PSRSSortRegularSample, key=125e3a7f78bcb6c939668f03a3b94787_0>,
 Chunk <op=PSRSConcatPivot, key=58ce89bb01132fd0fd757246ef654c40_0>,
 Chunk <op=PSRSSortRegularSample, key=98e601719051e980b4d9f3f20b20e437_1>,
 Chunk <op=PSRSSortRegularSample, key=125e3a7f78bcb6c939668f03a3b94787_1>,
 Chunk <op=PSRSSortRegularSample, key=98e601719051e980b4d9f3f20b20e437_0>,
 Chunk <op=TensorSlice, key=bdbeadf9202694d9c8f86b9fc8fc2db2_0>,
 Chunk <op=PSRSShuffle, stage=reduce, key=22974984dc0b50cff4d077e0413202d2_0>,
 Chunk <op=PSRSShuffle, stage=reduce, key=6c78a27cd13360463638ae139958297e_0>,
 Chunk <op=TensorShuffleProxy, key=fd341cca1ae125e359e9a2274928605f_0>,
 Chunk <op=PSRSShuffle, stage=map, key=7d9f9ea924cf86e2aba771257f46db95_0>,
 Chunk <op=PSRSShuffle, stage=map, key=45359c01dfab34dd6d552687e3e9cf86_0>,
 Chunk <op=PSRSSortRegularSample, key=cac04cc478ecb467c0f1a7bca4bf809c_0>,
 Chunk <op=PSRSConcatPivot, key=ee1f386a09653687d7f3802aaf6d8cd3_0>,
 Chunk <op=PSRSSortRegularSample, key=321f2eda3a690282eb2d75ea0aa2614e_1>,
 Chunk <op=PSRSSortRegularSample, key=cac04cc478ecb467c0f1a7bca4bf809c_1>,
 Chunk <op=PSRSSortRegularSample, key=321f2eda3a690282eb2d75ea0aa2614e_0>,
 Chunk <op=PSRSShuffle, stage=reduce, key=02d268f3091f3c95271acc314e0ef0fd_0>,
 Chunk <op=PSRSShuffle, stage=reduce, key=94c1f8fa627bcd5960ad795f7c692671_0>,
 Chunk <op=TensorShuffleProxy, key=6e7794b968faabcbebdfb289789c1ab9_0>,
 Chunk <op=PSRSShuffle, stage=map, key=0ebf73717ef2479456a6351c6209a90e_0>,
 Chunk <op=PSRSShuffle, stage=map, key=eb053dc8499bc7cdc12d8382f199e096_0>,
 Chunk <op=PSRSSortRegularSample, key=125e3a7f78bcb6c939668f03a3b94787_0>,
 Chunk <op=PSRSConcatPivot, key=58ce89bb01132fd0fd757246ef654c40_0>,
 Chunk <op=PSRSSortRegularSample, key=98e601719051e980b4d9f3f20b20e437_1>,
 Chunk <op=PSRSSortRegularSample, key=125e3a7f78bcb6c939668f03a3b94787_1>,
 Chunk <op=PSRSSortRegularSample, key=98e601719051e980b4d9f3f20b20e437_0>,
 Chunk <op=PSRSShuffle, stage=reduce, key=22974984dc0b50cff4d077e0413202d2_0>]
['94c1f8fa627bcd5960ad795f7c692671_0',
 '6e7794b968faabcbebdfb289789c1ab9_0',
 '0ebf73717ef2479456a6351c6209a90e_0',
 'eb053dc8499bc7cdc12d8382f199e096_0',
 '125e3a7f78bcb6c939668f03a3b94787_0',
 '58ce89bb01132fd0fd757246ef654c40_0',
 '98e601719051e980b4d9f3f20b20e437_1',
 '125e3a7f78bcb6c939668f03a3b94787_1',
 '98e601719051e980b4d9f3f20b20e437_0',
 '22974984dc0b50cff4d077e0413202d2_0']
```

Dump graph: [default.pdf](https://github.com/mars-project/mars/files/8645133/default.pdf)

I added check output code to the `PSRSShuffle._execute_map`:
```python
for i in range(op.n_partition):
    key = (out.key, (i,))
    value = tuple(reduce_outputs[i].ravel())
    if key in ctx:
        existing_value = ctx[key]
        np.testing.assert_array_equal(existing_value, value)
        raise Exception(f"The key already exists: {key}")
    ctx[out.key, (i,)] = tuple(reduce_outputs[i].ravel())
```

It raises an exception and the `assert_array_equal` passes.

```python
            for i in range(op.n_partition):
                key = (out.key, (i,))
                value = tuple(reduce_outputs[i].ravel())
                if key in ctx:
                    existing_value = ctx[key]
                    np.testing.assert_array_equal(existing_value, value)
>                   raise Exception(f"The key already exists: {key}")
E                   Exception: The key already exists: ('eb053dc8499bc7cdc12d8382f199e096_0', (0,))

mars/tensor/base/psrs.py:753: Exception
```

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version 3.7.7
2. The version of Mars you use Latest master
3. Versions of crucial packages, such as numpy, scipy and pandas
4. Full stack of the error.
5. Minimized code to reproduce the error.

**Expected behavior**
A clear and concise description of what you expected to happen.

**Additional context**
Add any other context about the problem here.



Discussion:

Resolving patch:
diff --git a/mars/core/entity/utils.py b/mars/core/entity/utils.py
--- a/mars/core/entity/utils.py
+++ b/mars/core/entity/utils.py
@@ -76,21 +76,24 @@ def recursive_tile(
     q = [t for t in to_tile if t.is_coarse()]
     while q:
         t = q[-1]
-        cs = [c for c in t.inputs if c.is_coarse()]
-        if cs:
-            q.extend(cs)
-            continue
-        for obj in handler.tile(t.op.outputs):
-            to_update_inputs = []
-            chunks = []
-            for inp in t.op.inputs:
-                chunks.extend(inp.chunks)
-                if has_unknown_shape(inp):
-                    to_update_inputs.append(inp)
-            if obj is None:
-                yield chunks + to_update_inputs
-            else:
-                yield obj + to_update_inputs
+        if t.is_coarse():
+            # t may be put into q repeatedly,
+            # so we check if it's tiled or not
+            cs = [c for c in t.inputs if c.is_coarse()]
+            if cs:
+                q.extend(cs)
+                continue
+            for obj in handler.tile(t.op.outputs):
+                to_update_inputs = []
+                chunks = []
+                for inp in t.op.inputs:
+                    chunks.extend(inp.chunks)
+                    if has_unknown_shape(inp):
+                        to_update_inputs.append(inp)
+                if obj is None:
+                    yield chunks + to_update_inputs
+                else:
+                    yield obj + to_update_inputs
         q.pop()
 
     if not return_list:


Test patch:
diff --git a/mars/core/entity/tests/test_utils.py b/mars/core/entity/tests/test_utils.py
--- a/mars/core/entity/tests/test_utils.py
+++ b/mars/core/entity/tests/test_utils.py
@@ -44,7 +44,7 @@ def test_recursive_tile(setup):
     d2 = mt.random.rand(10, chunk_size=5)
     op = _TestOperand()
     t = op.new_tensor([d1, d2], dtype=d1.dtype, shape=(20,), order=d1.order)
-    t.execute()
+    t.execute(extra_config={"check_duplicated_operand_keys": True})
 
 
 class _TestOperandWithDuplicatedSubmission(TensorOperand, TensorOperandMixin):
diff --git a/mars/services/task/supervisor/tests/task_preprocessor.py b/mars/services/task/supervisor/tests/task_preprocessor.py
--- a/mars/services/task/supervisor/tests/task_preprocessor.py
+++ b/mars/services/task/supervisor/tests/task_preprocessor.py
@@ -51,6 +51,9 @@ def __init__(self, *args, **kwargs):
         for key in _check_args:
             check_options[key] = kwargs.get(key, True)
         self._check_options = check_options
+        self._check_duplicated_operand_keys = bool(
+            kwargs.get("check_duplicated_operand_keys")
+        )
 
     def _get_done(self):
         return super()._get_done()
@@ -145,6 +148,13 @@ def analyze(
         stage_id: str,
         op_to_bands: Dict[str, BandType] = None,
     ) -> SubtaskGraph:
+        # check if duplicated operand keys exist
+        if self._check_duplicated_operand_keys and len(
+            {c.key for c in chunk_graph}
+        ) < len(
+            chunk_graph
+        ):  # pragma: no cover
+            raise AssertionError("Duplicated operands exist")
         # record shapes generated in tile
         for n in chunk_graph:
             self._raw_chunk_shapes[n.key] = getattr(n, "shape", None)