Repository: sql-machine-learning/elasticdl

Issue:
Should master fail if all workers failed ?
Sometimes, all workers fail due to errors of data or model codes. The master should fail and exit to show the job fails. The user may think the job is running if the master does not fail.


Discussion:

Resolving patch:
diff --git a/elasticdl/python/master/k8s_instance_manager.py b/elasticdl/python/master/k8s_instance_manager.py
--- a/elasticdl/python/master/k8s_instance_manager.py
+++ b/elasticdl/python/master/k8s_instance_manager.py
@@ -4,7 +4,7 @@
 from collections import Counter
 
 from elasticdl.python.common import k8s_client as k8s
-from elasticdl.python.common.constants import BashCommandTemplate
+from elasticdl.python.common.constants import BashCommandTemplate, PodStatus
 from elasticdl.python.common.log_utils import default_logger as logger
 
 _SERVICE_ADDR_SEP = ","
@@ -107,6 +107,7 @@ def __init__(
         self._relaunch_deleted_live_ps = True
 
         self._failed_pods = []
+        self.all_workers_failed = False
 
         if disable_relaunch:
             self._k8s_client = k8s.Client(**kwargs)
@@ -290,11 +291,12 @@ def _event_cb(self, event):
         with self._lock:
             if pod_name in self._failed_pods:
                 return
+
             # When a pod fails with exit_code == 137, it may be deleted,
             # preempted, or OOMkilled. Master will try to relaunch it.
             # For OOMkilled, the relaunch is a workaround for memory leak
             # issues in tf eager mode.
-            failed_pod = False
+            relaunch_failed_pod = False
             if (
                 evt_type == "MODIFIED"
                 and phase == "Failed"
@@ -306,7 +308,7 @@ def _event_cb(self, event):
                 == 137
             ):
                 self._failed_pods.append(pod_name)
-                failed_pod = True
+                relaunch_failed_pod = True
                 logger.info(
                     "Pod %s is killed with reason %s."
                     % (
@@ -316,10 +318,11 @@ def _event_cb(self, event):
                         ].state.terminated.reason,
                     )
                 )
+
             if pod_name in self._worker_pod_name_to_id:
                 worker_id = self._worker_pod_name_to_id.get(pod_name)
                 self._worker_pods_phase[worker_id] = (pod_name, phase)
-                if evt_type == "DELETED" or failed_pod:
+                if evt_type == "DELETED" or relaunch_failed_pod:
                     del self._worker_pods_phase[worker_id]
                     del self._worker_pod_name_to_id[pod_name]
                     self._task_d.recover_tasks(worker_id)
@@ -329,11 +332,16 @@ def _event_cb(self, event):
                         self._relaunch_deleted_live_worker
                         and phase != "Succeeded"
                     )
+                else:
+                    workers_failed = []
+                    for pod_name, phase in self._worker_pods_phase.values():
+                        workers_failed.append(phase == PodStatus.FAILED)
+                    self.all_workers_failed = all(workers_failed)
 
             elif pod_name in self._ps_pod_name_to_id:
                 ps_id = self._ps_pod_name_to_id.get(pod_name)
                 self._ps_pods_phase[ps_id] = (pod_name, phase)
-                if evt_type == "DELETED" or failed_pod:
+                if evt_type == "DELETED" or relaunch_failed_pod:
                     del self._ps_pods_phase[ps_id]
                     del self._ps_pod_name_to_id[pod_name]
                     relaunch_ps = self._relaunch_deleted_live_ps
diff --git a/elasticdl/python/master/master.py b/elasticdl/python/master/master.py
--- a/elasticdl/python/master/master.py
+++ b/elasticdl/python/master/master.py
@@ -223,6 +223,11 @@ def run(self):
         """
         try:
             while True:
+                if self.instance_manager.all_workers_failed:
+                    raise Exception(
+                        "All workers fail with unrecoverable errors"
+                    )
+                    break
                 if self.task_d.finished():
                     if self.instance_manager:
                         self.instance_manager.update_status(


Test patch: