Repository: mars-project/mars

Issue:
[BUG] Mars scheduler not responding under some conditions
Under conditions listed below, Mars scheduler will not respond when graph execution fails:

1. When graph tiling fails, scheduler log will report IndexError, while the state of the graph keeps RUNNING;
2. When a terminating chunk fails and the shape of corresponding tileable has NaNs, the state of the graph keeps.RUNNING.


Discussion:

Resolving patch:
diff --git a/mars/scheduler/graph.py b/mars/scheduler/graph.py
--- a/mars/scheduler/graph.py
+++ b/mars/scheduler/graph.py
@@ -433,7 +433,10 @@ def stop_graph(self):
 
     @log_unhandled
     def get_chunk_graph(self):
-        return self._chunk_graph_builder.iterative_chunk_graphs[-1]
+        try:
+            return self._chunk_graph_builder.iterative_chunk_graphs[-1]
+        except (IndexError, AttributeError):
+            raise GraphNotExists from None
 
     def _gen_tileable_graph(self):
         if self._tileable_graph_cache is None:
@@ -928,7 +931,12 @@ def add_finished_terminal(self, op_key, final_state=None, exc=None):
         :param op_key: operand key
         :param final_state: state of the operand
         """
+        if exc is not None:
+            self._graph_meta_ref.set_exc_info(exc, _tell=True, _wait=False)
+
         tileable_keys = self._terminal_chunk_op_key_to_tileable_key[op_key]
+        is_failed = final_state in (GraphState.CANCELLED, GraphState.FAILED)
+        terminal_tileable_count = len(self._terminal_tileable_key_to_chunk_op_keys)
         for tileable_key in tileable_keys:
             self._target_tileable_finished[tileable_key].add(op_key)
             if final_state == GraphState.FAILED:
@@ -936,11 +944,12 @@ def add_finished_terminal(self, op_key, final_state=None, exc=None):
                     self.final_state = GraphState.FAILED
             elif final_state == GraphState.CANCELLED:
                 self.final_state = final_state
+
             if self._target_tileable_finished[tileable_key] == \
                     self._terminal_tileable_key_to_chunk_op_keys[tileable_key]:
                 self._terminated_tileable_keys.add(tileable_key)
                 self._all_terminated_tileable_keys.add(tileable_key)
-                if len(self._terminated_tileable_keys) == len(self._terminal_tileable_key_to_chunk_op_keys):
+                if not is_failed and len(self._terminated_tileable_keys) == terminal_tileable_count:
                     # update shape if tileable or its chunks have unknown shape
                     self._update_tileable_and_its_chunk_shapes()
 
@@ -948,7 +957,7 @@ def add_finished_terminal(self, op_key, final_state=None, exc=None):
         self._terminated_chunk_keys.update([c.key for c in terminated_chunks
                                             if c.key in self._terminal_chunk_keys])
         if self._terminated_chunk_keys == self._terminal_chunk_keys:
-            if self._chunk_graph_builder.done:
+            if self._chunk_graph_builder.done or is_failed:
                 if self._chunk_graph_builder.prev_tileable_graph is not None:
                     # if failed before, clear intermediate data
                     to_free_tileable_keys = \
@@ -959,9 +968,6 @@ def add_finished_terminal(self, op_key, final_state=None, exc=None):
             else:
                 self._execute_graph(compose=self._chunk_graph_builder.is_compose)
 
-        if exc is not None:
-            self._graph_meta_ref.set_exc_info(exc, _tell=True, _wait=False)
-
     def _update_tileable_and_its_chunk_shapes(self):
         need_update_tileable_to_tiled = dict()
         for tileable in self._chunk_graph_builder.prev_tileable_graph:
diff --git a/mars/utils.py b/mars/utils.py
--- a/mars/utils.py
+++ b/mars/utils.py
@@ -317,7 +317,10 @@ def get_shuffle_input_keys_idxes(chunk):
 
 def _get_mod_logger():
     mod_logger = None
-    frame_globals = inspect.currentframe().f_back.f_globals
+    cur_frame = inspect.currentframe()
+    while cur_frame.f_globals.get('__name__') == __name__:
+        cur_frame = cur_frame.f_back
+    frame_globals = cur_frame.f_globals
     for logger_name in ('logger', 'LOG', 'LOGGER'):
         if logger_name in frame_globals:
             mod_logger = frame_globals[logger_name]


Test patch:
diff --git a/mars/deploy/local/tests/test_cluster.py b/mars/deploy/local/tests/test_cluster.py
--- a/mars/deploy/local/tests/test_cluster.py
+++ b/mars/deploy/local/tests/test_cluster.py
@@ -17,6 +17,7 @@
 import functools
 import logging
 import os
+import pickle
 import sys
 import time
 import tempfile
@@ -38,7 +39,7 @@
 from mars.config import options, option_context
 from mars.deploy.local.core import new_cluster, LocalDistributedCluster, gen_endpoint
 from mars.errors import ExecutionFailed
-from mars.serialize import Int64Field
+from mars.serialize import BytesField, Int64Field
 from mars.tensor.operands import TensorOperand
 from mars.tensor.arithmetic.core import TensorElementWise
 from mars.tensor.arithmetic.abs import TensorAbs
@@ -68,7 +69,38 @@ def __init__(self, f=None, **kw):
         super().__init__(_f=f, **kw)
 
 
-class FakeOp(TensorAbs):
+class TileFailOperand(TensorAbs):
+    _op_type_ = 198732951
+
+    _exc_serial = BytesField('exc_serial')
+
+    @classmethod
+    def tile(cls, op):
+        if op._exc_serial is not None:
+            raise pickle.loads(op._exc_serial)
+        return super().tile(op)
+
+
+class ExecFailOperand(TensorAbs):
+    _op_type_ = 196432154
+
+    _exc_serial = BytesField('exc_serial')
+
+    @classmethod
+    def tile(cls, op):
+        tileables = super().tile(op)
+        # make sure chunks
+        tileables[0]._shape = (np.nan, np.nan)
+        return tileables
+
+    @classmethod
+    def execute(cls, ctx, op):
+        if op._exc_serial is not None:
+            raise pickle.loads(op._exc_serial)
+        return super().execute(ctx, op)
+
+
+class TileWithContextOperand(TensorAbs):
     _op_type_ = 9870102948
 
     _multiplier = Int64Field('multiplier')
@@ -408,11 +440,27 @@ def testGraphFail(self, *_):
         op = SerializeMustFailOperand(f=3)
         tensor = op.new_tensor(None, (3, 3))
 
+        try:
+            raise ValueError
+        except:
+            exc = sys.exc_info()[1]
+
         with new_cluster(scheduler_n_process=2, worker_n_process=2,
-                         shared_memory='20M') as cluster:
+                         shared_memory='20M', modules=[__name__],
+                         options={'scheduler.retry_num': 1}) as cluster:
             with self.assertRaises(ExecutionFailed):
                 cluster.session.run(tensor, timeout=_exec_timeout)
 
+            data = mt.tensor(np.random.rand(10, 20))
+            data2 = TileFailOperand(_exc_serial=pickle.dumps(exc)).new_tensor([data], shape=data.shape)
+            with self.assertRaises(ExecutionFailed):
+                cluster.session.run(data2)
+
+            data = mt.tensor(np.random.rand(20, 10))
+            data2 = ExecFailOperand(_exc_serial=pickle.dumps(exc)).new_tensor([data], shape=data.shape)
+            with self.assertRaises(ExecutionFailed):
+                cluster.session.run(data2)
+
     def testFetchTensor(self, *_):
         with new_cluster(scheduler_n_process=2, worker_n_process=2,
                          shared_memory='20M', web=True) as cluster:
@@ -917,7 +965,7 @@ def testTileContextInLocalCluster(self):
 
             session.run(data)
 
-            data2 = FakeOp().new_tensor([data], shape=data.shape)
+            data2 = TileWithContextOperand().new_tensor([data], shape=data.shape)
 
             result = session.run(data2)
             np.testing.assert_array_equal(raw * raw.nbytes, result)