Repository: sql-machine-learning/elasticdl

Issue:
create_namespaced_pod failure would crash master pod
If `create_namespaced_pod` fails, it may crash the master pod.
Need to catch exception for this call.

```
Traceback (most recent call last):
  File "/usr/lib/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/usr/local/lib/python3.6/dist-packages/elasticdl/python/master/main.py", line 28, in <module>
    sys.exit(main())
  File "/usr/local/lib/python3.6/dist-packages/elasticdl/python/master/main.py", line 23, in main
    master.prepare()
  File "/usr/local/lib/python3.6/dist-packages/elasticdl/python/master/master.py", line 224, in prepare
    self.instance_manager.start_workers()
  File "/usr/local/lib/python3.6/dist-packages/elasticdl/python/master/k8s_instance_manager.py", line 210, in start_workers
    self._start_worker(self._next_worker_id())
  File "/usr/local/lib/python3.6/dist-packages/elasticdl/python/master/k8s_instance_manager.py", line 163, in _start_worker
    envs=copy.deepcopy(self._envs),
  File "/usr/local/lib/python3.6/dist-packages/elasticdl/python/common/k8s_client.py", line 175, in create_worker
    pod_name, "worker", kargs["worker_id"], **kargs
  File "/usr/local/lib/python3.6/dist-packages/elasticdl/python/common/k8s_client.py", line 170, in _create_ps_worker_pod
    return self.client.create_namespaced_pod(self.namespace, pod)
  File "/usr/local/lib/python3.6/dist-packages/kubernetes/client/apis/core_v1_api.py", line 6115, in create_namespaced_pod
    (data) = self.create_namespaced_pod_with_http_info(namespace, body, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/kubernetes/client/apis/core_v1_api.py", line 6206, in create_namespaced_pod_with_http_info
    collection_formats=collection_formats)
  File "/usr/local/lib/python3.6/dist-packages/kubernetes/client/api_client.py", line 344, in call_api
    _return_http_data_only, collection_formats, _preload_content, _request_timeout)
  File "/usr/local/lib/python3.6/dist-packages/kubernetes/client/api_client.py", line 178, in __call_api
    _request_timeout=_request_timeout)
  File "/usr/local/lib/python3.6/dist-packages/kubernetes/client/api_client.py", line 387, in request
    body=body)
  File "/usr/local/lib/python3.6/dist-packages/kubernetes/client/rest.py", line 266, in POST
    body=body)
  File "/usr/local/lib/python3.6/dist-packages/kubernetes/client/rest.py", line 222, in request
    raise ApiException(http_resp=r)
kubernetes.client.rest.ApiException: (403)
Reason: Forbidden
HTTP response headers: HTTPHeaderDict({'Audit-Id': '866b37f9-e7c7-443e-83fe-074527e4e34b', 'Content-Type': 'application/json', 'Date': 'Tue, 22 Sep 2020 09:46:50 GMT', 'Content-Length': '449'})
HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":" '****","code":403}
```


Discussion:

Resolving patch:
diff --git a/elasticdl/python/common/k8s_client.py b/elasticdl/python/common/k8s_client.py
--- a/elasticdl/python/common/k8s_client.py
+++ b/elasticdl/python/common/k8s_client.py
@@ -45,6 +45,7 @@ def __init__(
         namespace,
         job_name,
         event_callback=None,
+        periodic_call_func=None,
         cluster_spec="",
         force_use_kube_config_file=False
     ):
@@ -59,6 +60,7 @@ def __init__(
                 Used as pod name prefix and value for "elasticdl" label.
             event_callback: If not None, an event watcher will be created and
                 events passed to the callback.
+            periodic_call_func: If not None, call this method periodically.
             force_use_kube_config_file: If true, force to load the cluster
                 config from ~/.kube/config. Otherwise, if it's in a process
                 running in a K8S environment, it loads the incluster config,
@@ -72,10 +74,15 @@ def __init__(
             force_use_kube_config_file=force_use_kube_config_file,
         )
         self._event_cb = event_callback
+        self._periodic_call_func = periodic_call_func
         if self._event_cb:
             threading.Thread(
                 target=self._watch, name="event_watcher", daemon=True
             ).start()
+        if self._periodic_call_func:
+            threading.Thread(
+                target=self._periodic_call, name="periodic_call", daemon=True
+            ).start()
 
     def _watch(self):
         while True:
@@ -93,6 +100,11 @@ def _watch(self):
             # time and retry.
             time.sleep(5)
 
+    def _periodic_call(self):
+        while True:
+            self._periodic_call_func()
+            time.sleep(15)
+
     def _get_service_address(self, service_name, port):
         return "%s.%s.svc:%d" % (service_name, self.namespace, port)
 
@@ -167,7 +179,11 @@ def _create_ps_worker_pod(self, pod_name, type_key, index_key, **kargs):
         # Add replica type and index
         pod.metadata.labels[ELASTICDL_REPLICA_TYPE_KEY] = type_key
         pod.metadata.labels[ELASTICDL_REPLICA_INDEX_KEY] = str(index_key)
-        return self.client.create_namespaced_pod(self.namespace, pod)
+        try:
+            return self.client.create_namespaced_pod(self.namespace, pod)
+        except client.rest.ApiException as e:
+            logger.warning("Failed to create %s pod: %s\n" % (pod_name, e))
+            return None
 
     def create_worker(self, **kargs):
         pod_name = self.get_worker_pod_name(kargs["worker_id"])
diff --git a/elasticdl/python/master/k8s_instance_manager.py b/elasticdl/python/master/k8s_instance_manager.py
--- a/elasticdl/python/master/k8s_instance_manager.py
+++ b/elasticdl/python/master/k8s_instance_manager.py
@@ -14,6 +14,7 @@
 import copy
 import itertools
 import threading
+import time
 from collections import Counter
 
 from elasticdl.python.common import k8s_client as k8s
@@ -101,6 +102,23 @@ def __init__(
 
         # Protects followed variables, which are accessed from event_cb.
         self._lock = threading.Lock()
+
+        self._init_worker_pod_status()
+
+        if disable_relaunch:
+            self._k8s_client = k8s.Client(**kwargs)
+        else:
+            self._k8s_client = k8s.Client(
+                event_callback=self._event_cb,
+                periodic_call_func=self._process_worker,
+                **kwargs
+            )
+        self._ps_addrs = self._get_addrs(
+            self._num_ps, self._k8s_client.get_ps_service_address
+        )
+        self._worker_addrs = []
+
+    def _init_worker_pod_status(self):
         # worker id to (pod name, ip, phase) mapping
         # phase: None/Pending/Running/Succeeded/Failed/Unknown
         #   None: worker was just launched, haven't received event yet.
@@ -113,6 +131,8 @@ def __init__(
         self._worker_pods_ip_phase = {}
         # pod name to worker id mapping
         self._worker_pod_name_to_id = {}
+        # worker ids whose pods are not created
+        self._not_created_worker_id = []
 
         self._relaunch_deleted_live_worker = True
 
@@ -123,16 +143,12 @@ def __init__(
         self._failed_pods = []
         self.all_workers_failed = False
 
-        if disable_relaunch:
-            self._k8s_client = k8s.Client(**kwargs)
-        else:
-            self._k8s_client = k8s.Client(
-                event_callback=self._event_cb, **kwargs
-            )
-        self._ps_addrs = self._get_addrs(
-            self._num_ps, self._k8s_client.get_ps_service_address
-        )
-        self._worker_addrs = []
+    def _process_worker(self):
+        need_process = True
+        while need_process and self._not_created_worker_id:
+            worker_id = self._not_created_worker_id.pop()
+            # Try to create a worker pod with id as worker_id
+            need_process = self._start_worker(worker_id)
 
     def _start_worker(self, worker_id):
         logger.info("Starting worker: %d" % worker_id)
@@ -162,9 +178,14 @@ def _start_worker(self, worker_id):
                 ps_addrs=self._ps_addrs,
                 envs=copy.deepcopy(self._envs),
             )
-            name = pod.metadata.name
-            self._worker_pod_name_to_id[name] = worker_id
-            self._worker_pods_ip_phase[worker_id] = (name, None, None)
+            if pod:
+                name = pod.metadata.name
+                self._worker_pod_name_to_id[name] = worker_id
+                self._worker_pods_ip_phase[worker_id] = (name, None, None)
+                return True
+            else:
+                self._not_created_worker_id.append(worker_id)
+                return False
 
     def _start_ps(self, ps_id):
         logger.info("Starting PS: %d" % ps_id)
@@ -176,23 +197,32 @@ def _start_ps(self, ps_id):
             )
         ps_args = [self._ps_args[0], bash_command]
         with self._lock:
-            pod = self._k8s_client.create_ps(
-                ps_id=ps_id,
-                resource_requests=self._ps_resource_request,
-                resource_limits=self._ps_resource_limit,
-                pod_priority=self._ps_pod_priority,
-                volume=self._volume,
-                image_pull_policy=self._image_pull_policy,
-                command=self._ps_command,
-                args=ps_args,
-                restart_policy=self._restart_policy,
-                envs=copy.deepcopy(self._envs),
-            )
+            while True:
+                pod = self._create_ps_pod(ps_id, ps_args)
+                if pod:
+                    break
+                # TODO: should we fail the job when ps pods fail to
+                #       create for a long time?
+                time.sleep(15)
             name = pod.metadata.name
             self._ps_pod_name_to_id[name] = ps_id
             self._ps_pods_phase[ps_id] = (name, None)
             self._k8s_client.create_ps_service(ps_id)
 
+    def _create_ps_pod(self, ps_id, ps_args):
+        return self._k8s_client.create_ps(
+            ps_id=ps_id,
+            resource_requests=self._ps_resource_request,
+            resource_limits=self._ps_resource_limit,
+            pod_priority=self._ps_pod_priority,
+            volume=self._volume,
+            image_pull_policy=self._image_pull_policy,
+            command=self._ps_command,
+            args=ps_args,
+            restart_policy=self._restart_policy,
+            envs=copy.deepcopy(self._envs),
+        )
+
     def _get_addrs(self, num_addrs, addr_get_fn):
         addrs = []
         for addr_id in range(num_addrs):
@@ -232,27 +262,33 @@ def _remove_ps(self, ps_id):
 
         self._k8s_client.delete_ps(ps_id)
 
-    def stop_relaunch_and_remove_workers(self):
-        with self._lock:
+    def stop_relaunch_and_remove_pods(self, pod_type):
+        ids = []
+        if pod_type == "worker":
             self._relaunch_deleted_live_worker = False
-            for worker_id in self._worker_pods_ip_phase:
-                self._k8s_client.delete_worker(worker_id)
-
-    def stop_relaunch_and_remove_all_ps(self):
-        with self._lock:
+            ids = self._worker_pods_ip_phase
+            delete_func = self._k8s_client.delete_worker
+        elif pod_type == "ps":
             self._relaunch_deleted_live_ps = False
-            for ps_id in self._ps_pods_phase:
-                self._k8s_client.delete_ps(ps_id)
+            ids = self._ps_pods_phase
+            delete_func = self._k8s_client.delete_ps
+        with self._lock:
+            for id in ids:
+                delete_func(id)
 
-    def get_worker_counter(self):
+    def get_pod_counter(self, pod_type):
         with self._lock:
-            return Counter(
+            worker_counter = Counter(
                 [v for _, _, v in self._worker_pods_ip_phase.values()]
             )
+            ps_counter = Counter([v for _, v in self._ps_pods_phase.values()])
 
-    def get_ps_counter(self):
-        with self._lock:
-            return Counter([v for _, v in self._ps_pods_phase.values()])
+        if pod_type == "worker":
+            return worker_counter
+        elif pod_type == "ps":
+            return ps_counter
+        else:
+            return None
 
     def _event_cb(self, event):
         evt_obj = event.get("object")


Test patch:
diff --git a/elasticdl/python/tests/k8s_instance_manager_test.py b/elasticdl/python/tests/k8s_instance_manager_test.py
--- a/elasticdl/python/tests/k8s_instance_manager_test.py
+++ b/elasticdl/python/tests/k8s_instance_manager_test.py
@@ -37,21 +37,30 @@ def test_create_delete_worker_pod(self):
             worker_command=["/bin/bash"],
             worker_args=["-c", "echo"],
             namespace="default",
-            num_workers=3,
+            num_workers=2,
         )
 
         instance_manager.start_workers()
         max_check_num = 20
         for _ in range(max_check_num):
             time.sleep(3)
-            counters = instance_manager.get_worker_counter()
+            counters = instance_manager.get_pod_counter(pod_type="worker")
+            if counters["Succeeded"] == 2:
+                break
+
+        instance_manager._not_created_worker_id = [2]
+        instance_manager._worker_pod_priority[2] = None
+        instance_manager._process_worker()
+        for _ in range(max_check_num):
+            time.sleep(3)
+            counters = instance_manager.get_pod_counter(pod_type="worker")
             if counters["Succeeded"] == 3:
                 break
 
-        instance_manager.stop_relaunch_and_remove_workers()
+        instance_manager.stop_relaunch_and_remove_pods(pod_type="worker")
         for _ in range(max_check_num):
             time.sleep(3)
-            counters = instance_manager.get_worker_counter()
+            counters = instance_manager.get_pod_counter(pod_type="worker")
             if not counters:
                 break
         self.assertFalse(counters)
@@ -77,12 +86,12 @@ def test_get_worker_addrs(self):
         max_check_num = 20
         for _ in range(max_check_num):
             time.sleep(3)
-            counters = instance_manager.get_worker_counter()
+            counters = instance_manager.get_pod_counter(pod_type="worker")
             if counters["Running"]:
                 worker_addrs = instance_manager._get_alive_worker_addr()
                 self.assertEqual(len(worker_addrs), counters["Running"])
 
-        instance_manager.stop_relaunch_and_remove_workers()
+        instance_manager.stop_relaunch_and_remove_pods(pod_type="worker")
 
     @unittest.skipIf(
         os.environ.get("K8S_TESTS", "True") == "False",
@@ -110,14 +119,14 @@ def test_failed_worker_pod(self):
         max_check_num = 20
         for _ in range(max_check_num):
             time.sleep(3)
-            counters = instance_manager.get_worker_counter()
+            counters = instance_manager.get_pod_counter(pod_type="worker")
             if counters["Failed"] == 3:
                 break
 
-        instance_manager.stop_relaunch_and_remove_workers()
+        instance_manager.stop_relaunch_and_remove_pods(pod_type="worker")
         for _ in range(max_check_num):
             time.sleep(3)
-            counters = instance_manager.get_worker_counter()
+            counters = instance_manager.get_pod_counter(pod_type="worker")
             if not counters:
                 break
         task_d.recover_tasks.assert_has_calls(
@@ -147,7 +156,7 @@ def test_relaunch_worker_pod(self):
         max_check_num = 60
         for _ in range(max_check_num):
             time.sleep(1)
-            counters = instance_manager.get_worker_counter()
+            counters = instance_manager.get_pod_counter(pod_type="worker")
             if counters["Running"] + counters["Pending"] > 0:
                 break
         # Note: There is a slight chance of race condition.
@@ -178,7 +187,7 @@ def test_relaunch_worker_pod(self):
         else:
             self.fail("Failed to find newly launched worker.")
 
-        instance_manager.stop_relaunch_and_remove_workers()
+        instance_manager.stop_relaunch_and_remove_pods(pod_type="worker")
 
     @unittest.skipIf(
         os.environ.get("K8S_TESTS", "True") == "False",
@@ -212,7 +221,7 @@ def test_relaunch_ps_pod(self):
         max_check_num = 60
         for _ in range(max_check_num):
             time.sleep(1)
-            counters = instance_manager.get_ps_counter()
+            counters = instance_manager.get_pod_counter(pod_type="ps")
             if counters["Running"] + counters["Pending"] > 0:
                 break
         # Note: There is a slight chance of race condition.
@@ -242,7 +251,7 @@ def test_relaunch_ps_pod(self):
         else:
             self.fail("Failed to find newly launched ps.")
 
-        instance_manager.stop_relaunch_and_remove_all_ps()
+        instance_manager.stop_relaunch_and_remove_pods(pod_type="ps")
 
 
 if __name__ == "__main__":