Repository: sql-machine-learning/elasticdl

Issue:
Master should exit if all workers completed
![image](https://user-images.githubusercontent.com/18071380/90867462-fc5a6100-e3c7-11ea-8988-22fb42f75998.png)



Discussion:
The error happens when some task fails because of timeout of workers.
![image](https://user-images.githubusercontent.com/18071380/93039568-6b9b3c00-f67a-11ea-8f2e-d4de1a2409dd.png)

The worker cannot create dataset when there are pending tasks.
![image](https://user-images.githubusercontent.com/18071380/93039645-a309e880-f67a-11ea-906a-fba2821352eb.png)


Resolving patch:
diff --git a/elasticdl/python/master/k8s_instance_manager.py b/elasticdl/python/master/k8s_instance_manager.py
--- a/elasticdl/python/master/k8s_instance_manager.py
+++ b/elasticdl/python/master/k8s_instance_manager.py
@@ -142,7 +142,7 @@ def _init_worker_pod_status(self):
         self._relaunch_deleted_live_ps = True
 
         self._failed_pods = []
-        self.all_workers_failed = False
+        self.all_workers_exited = False
 
     def _process_worker(self):
         need_process = True
@@ -365,14 +365,7 @@ def _event_cb(self, event):
                         and phase != "Succeeded"
                     )
                 else:
-                    workers_failed = []
-                    for (
-                        pod_name,
-                        _,
-                        phase,
-                    ) in self._worker_pods_ip_phase.values():
-                        workers_failed.append(phase == PodStatus.FAILED)
-                    self.all_workers_failed = all(workers_failed)
+                    self.check_all_worker_exited()
 
             elif pod_name in self._ps_pod_name_to_id:
                 ps_id = self._ps_pod_name_to_id.get(pod_name)
@@ -404,6 +397,14 @@ def _event_cb(self, event):
             # tolerance.
             self._start_ps(ps_id)
 
+    def check_all_worker_exited(self):
+        workers_exited = []
+        for (pod_name, _, phase,) in self._worker_pods_ip_phase.values():
+            workers_exited.append(
+                phase == PodStatus.FAILED or phase == PodStatus.SUCCEEDED
+            )
+        self.all_workers_exited = all(workers_exited)
+
     def get_alive_workers(self):
         alive_workers = []
         for pod_name, _, phase in self._worker_pods_ip_phase.values():
diff --git a/elasticdl/python/master/master.py b/elasticdl/python/master/master.py
--- a/elasticdl/python/master/master.py
+++ b/elasticdl/python/master/master.py
@@ -225,17 +225,18 @@ def run(self):
         """
         try:
             while True:
-                if self.instance_manager.all_workers_failed:
-                    raise Exception(
-                        "All workers fail with unrecoverable errors"
-                    )
-                    break
                 if self.task_d.finished():
                     if self.instance_manager:
                         self.instance_manager.update_status(
                             InstanceManagerStatus.FINISHED
                         )
                     break
+                if self.instance_manager.all_workers_exited:
+                    raise Exception(
+                        "All workers exited but there also are",
+                        "unfinished tasks",
+                    )
+                    break
                 if self._should_stop:
                     break
                 time.sleep(30)


Test patch: