Repository: mars-project/mars

Issue:
[BUG] `testIterativeTilingWithoutEtcd` may fail sometimes for Python 3.5 in CI
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**

`testIterativeTilingWithoutEtcd` may fail sometimes for Python 3.5 in CI.

Error message is shown below.

```
CH0 2020-01-03 21:23:06,273 Unexpected exception occurred in enter_build_mode.<locals>.inner.
Traceback (most recent call last):
  File "/Users/xuyeqin/Workspace/mars/mars/utils.py", line 332, in _wrapped
    return func(*args, **kwargs)
  File "/Users/xuyeqin/Workspace/mars/mars/utils.py", line 465, in inner
    return func(*args, **kwargs)
  File "/Users/xuyeqin/Workspace/mars/mars/scheduler/graph.py", line 626, in prepare_graph
    tileable_graph=to_run_tileable_graph)
  File "/Users/xuyeqin/Workspace/mars/mars/utils.py", line 378, in _wrapped
    return func(*args, **kwargs)
  File "/Users/xuyeqin/Workspace/mars/mars/utils.py", line 465, in inner
    return func(*args, **kwargs)
  File "/Users/xuyeqin/Workspace/mars/mars/tiles.py", line 340, in build
    tileables, tileable_graph=tileable_graph)
  File "/Users/xuyeqin/Workspace/mars/mars/utils.py", line 378, in _wrapped
    return func(*args, **kwargs)
  File "/Users/xuyeqin/Workspace/mars/mars/utils.py", line 465, in inner
    return func(*args, **kwargs)
  File "/Users/xuyeqin/Workspace/mars/mars/tiles.py", line 254, in build
    self._on_tile_failure(tileable_data.op, exc_info)
  File "/Users/xuyeqin/Workspace/mars/mars/tiles.py", line 292, in inner
    raise exc_info[1].with_traceback(exc_info[2]) from None
  File "/Users/xuyeqin/Workspace/mars/mars/tiles.py", line 234, in build
    tiled = self._tile(tileable_data, tileable_graph)
  File "/Users/xuyeqin/Workspace/mars/mars/tiles.py", line 328, in _tile
    return super()._tile(tileable_data, tileable_graph)
  File "/Users/xuyeqin/Workspace/mars/mars/tiles.py", line 194, in _tile
    tds = on_tile(tileable_data.op.outputs, tds)
  File "/Users/xuyeqin/Workspace/mars/mars/scheduler/graph.py", line 568, in on_tile
    tiled_inp = get_tiled(self._tileable_to_fetch[inp])
  File "/Users/xuyeqin/Workspace/mars/mars/utils.py", line 465, in inner
    return func(*args, **kwargs)
  File "/Users/xuyeqin/Workspace/mars/mars/tiles.py", line 150, in get_tiled
    return _tileable_data_to_tiled[tileable_data]
  File "/Users/xuyeqin/miniconda3/envs/mars/lib/python3.5/weakref.py", line 394, in __getitem__
    return self.data[ref(key)]
KeyError: <weakref at 0x7fe6705edb38; to 'TensorData' at 0x7fe6705ce610>
```


Discussion:

Resolving patch:
diff --git a/mars/scheduler/graph.py b/mars/scheduler/graph.py
--- a/mars/scheduler/graph.py
+++ b/mars/scheduler/graph.py
@@ -609,8 +609,11 @@ def on_tile_success(tiled_before, tiled_after):
             for failed_op in chunk_graph_builder.interrupted_ops:
                 for inp in failed_op.inputs:
                     if inp not in failed_tileable_set:
-                        fetch_inp = build_fetch_tileable(inp).data
-                        self._tileable_to_fetch[inp] = fetch_inp
+                        if inp not in self._tileable_to_fetch:
+                            fetch_inp = build_fetch_tileable(inp).data
+                            self._tileable_to_fetch[inp] = fetch_inp
+                        else:
+                            fetch_inp = self._tileable_to_fetch[inp]
                         to_fetch_tileables.append(fetch_inp)
                         to_run_tileable_graph.add_node(fetch_inp)
                         for o in failed_op.outputs:
@@ -969,7 +972,7 @@ def _update_tileable_and_its_chunk_shapes(self):
             chunk.data._shape = chunk_meta.chunk_shape
 
         for tileable, tiled in need_update_tileable_to_tiled.items():
-            chunk_idx_to_shape = {c.index: c.shape for c in tiled.chunks}
+            chunk_idx_to_shape = OrderedDict((c.index, c.shape) for c in tiled.chunks)
             nsplits = calc_nsplits(chunk_idx_to_shape)
             tiled._nsplits = nsplits
             if any(np.isnan(s) for s in tileable.shape):
@@ -1119,6 +1122,8 @@ def fetch_tileable_result(self, tileable_key, _check=True):
         ctx = dict()
         for chunk in tileable.chunks:
             endpoints = self.chunk_meta.get_workers(self._session_id, chunk.key)
+            if endpoints is None:
+                raise KeyError('cannot fetch meta of chunk {}'.format(chunk))
             sender_ref = self.ctx.actor_ref(ResultSenderActor.default_uid(), address=endpoints[-1])
             ctx[chunk.key] = dataserializer.loads(sender_ref.fetch_data(self._session_id, chunk.key))
 


Test patch:
diff --git a/mars/scheduler/tests/integrated/test_normal_execution.py b/mars/scheduler/tests/integrated/test_normal_execution.py
--- a/mars/scheduler/tests/integrated/test_normal_execution.py
+++ b/mars/scheduler/tests/integrated/test_normal_execution.py
@@ -294,7 +294,7 @@ def testIterativeTilingWithoutEtcd(self):
         expected = np.sort(raw)[:5]
         assert_allclose(loads(result), expected)
 
-        with self.assertRaises(TypeError):
+        with self.assertRaises(KeyError):
             session_ref.fetch_result(graph_key, a.key, check=False)
 
         raw1 = rs.rand(20)