Repository: sql-machine-learning/elasticdl

Issue:
Event callback should not check the existence of master pod
Currently, there's error log when event callback checks the pod existence like this `Unknown pod name: elasticdl-master-test-mnist`. This is because `elasticdl-master-test-mnist` is the master pod that's created by users instead of being created by `WorkerManager`. We should fix this since it should only check pod existence for worker pod.

Complete log:
```
2019-06-06 17:38:37,952 elasticdl.python.elasticdl.master.k8s_worker_manager INFO     [k8s_worker_manager.py:121] Got event ADDED, phase Running for pod: elasticdl-master-test-mnist
2019-06-06 17:38:37,953 elasticdl.python.elasticdl.master.k8s_worker_manager INFO     [k8s_worker_manager.py:71] Starting worker: 1
2019-06-06 17:38:37,953 elasticdl.python.elasticdl.master.k8s_worker_manager ERROR    [k8s_worker_manager.py:127] Unknown pod name: elasticdl-master-test-mnist
2019-06-06 17:38:37,954 elasticdl.python.elasticdl.master.k8s_client INFO     [k8s_client.py:105] Creating worker: 1
2019-06-06 17:38:37,955 elasticdl.python.elasticdl.master.k8s_worker_manager INFO     [k8s_worker_manager.py:121] Got event ADDED, phase Pending for pod: elasticdl-worker-test-mnist-0
```
Event callback should not check the existence of master pod
Currently, there's error log when event callback checks the pod existence like this `Unknown pod name: elasticdl-master-test-mnist`. This is because `elasticdl-master-test-mnist` is the master pod that's created by users instead of being created by `WorkerManager`. We should fix this since it should only check pod existence for worker pod.

Complete log:
```
2019-06-06 17:38:37,952 elasticdl.python.elasticdl.master.k8s_worker_manager INFO     [k8s_worker_manager.py:121] Got event ADDED, phase Running for pod: elasticdl-master-test-mnist
2019-06-06 17:38:37,953 elasticdl.python.elasticdl.master.k8s_worker_manager INFO     [k8s_worker_manager.py:71] Starting worker: 1
2019-06-06 17:38:37,953 elasticdl.python.elasticdl.master.k8s_worker_manager ERROR    [k8s_worker_manager.py:127] Unknown pod name: elasticdl-master-test-mnist
2019-06-06 17:38:37,954 elasticdl.python.elasticdl.master.k8s_client INFO     [k8s_client.py:105] Creating worker: 1
2019-06-06 17:38:37,955 elasticdl.python.elasticdl.master.k8s_worker_manager INFO     [k8s_worker_manager.py:121] Got event ADDED, phase Pending for pod: elasticdl-worker-test-mnist-0
```


Discussion:

Resolving patch:
diff --git a/elasticdl/python/elasticdl/master/k8s_client.py b/elasticdl/python/elasticdl/master/k8s_client.py
--- a/elasticdl/python/elasticdl/master/k8s_client.py
+++ b/elasticdl/python/elasticdl/master/k8s_client.py
@@ -5,6 +5,8 @@
 
 from kubernetes import client, config, watch
 
+WORKER_POD_NAME_PREFIX = "elasticdl-worker-"
+
 
 class Client(object):
     def __init__(
@@ -51,14 +53,14 @@ def _watch(self):
             except Exception:
                 traceback.print_exc()
 
-    def get_pod_name(self, worker_id):
-        return "elasticdl-worker-" + self._job_name + "-" + str(worker_id)
+    def get_worker_pod_name(self, worker_id):
+        return WORKER_POD_NAME_PREFIX + self._job_name + "-" + str(worker_id)
 
     def _create_worker_pod(self, worker_id, resource_requests, resource_limits, priority,
                            mount_path, volume_name, image_pull_policy, command, args, restart_policy):
         # Worker container config
         container = client.V1Container(
-            name=self.get_pod_name(worker_id),
+            name=self.get_worker_pod_name(worker_id),
             image=self._image,
             command=command,
             resources=client.V1ResourceRequirements(
@@ -90,7 +92,7 @@ def _create_worker_pod(self, worker_id, resource_requests, resource_limits, prio
         pod = client.V1Pod(
             spec=spec,
             metadata=client.V1ObjectMeta(
-                name=self.get_pod_name(worker_id),
+                name=self.get_worker_pod_name(worker_id),
                 labels={
                     "app": "elasticdl",
                     "elasticdl_job_name": self._job_name
@@ -112,7 +114,7 @@ def create_worker(self, worker_id, resource_requests, resource_limits, priority=
     def delete_worker(self, worker_id):
         self._logger.info("Deleting worker: " + str(worker_id))
         self._v1.delete_namespaced_pod(
-            self.get_pod_name(worker_id),
+            self.get_worker_pod_name(worker_id),
             self._ns,
             body=client.V1DeleteOptions(grace_period_seconds=0),
         )
diff --git a/elasticdl/python/elasticdl/master/k8s_worker_manager.py b/elasticdl/python/elasticdl/master/k8s_worker_manager.py
--- a/elasticdl/python/elasticdl/master/k8s_worker_manager.py
+++ b/elasticdl/python/elasticdl/master/k8s_worker_manager.py
@@ -123,8 +123,8 @@ def _event_cb(self, event):
         relaunch = False
         with self._lock:
             worker_id = self._pod_name_to_id.get(pod_name)
-            if worker_id is None:
-                self._logger.error("Unknown pod name: %s" % pod_name)
+            if worker_id is None and k8s.WORKER_POD_NAME_PREFIX in pod_name:
+                self._logger.error("Unknown worker pod name: %s" % pod_name)
                 return
 
             self._pods_phase[worker_id] = (pod_name, phase)


Test patch: