Repository: sql-machine-learning/elasticdl

Issue:
service domain name will be changed after relaunch due to PR 1543
ElasticDL workers use service domain names to connect to PS pods. 
Service domain names should not be changed after PS pods relaunch.
https://github.com/sql-machine-learning/elasticdl/pull/1543 causes a regression that will change the service domain name after PS relaunch.

```
[2019-12-05 11:31:19,096] [INFO] [k8s_instance_manager.py:139:_update_addr] addr list before update is ['elasticdl-test-embedding-edl-ps-0.kubemaker.svc:2222', 'elasticdl-test-embedding-edl-ps-1.kubemaker.svc:2222']
[2019-12-05 11:31:19,096] [INFO] [k8s_instance_manager.py:143:_update_addr] addr list after update is ['elasticdl-test-embedding-edl-ps-0.kubemaker.svc:2222', 'elasticdl-test-embedding-edl-ps-2.kubemaker.svc:2222']
```


Discussion:

Resolving patch:
diff --git a/elasticdl/python/master/k8s_instance_manager.py b/elasticdl/python/master/k8s_instance_manager.py
--- a/elasticdl/python/master/k8s_instance_manager.py
+++ b/elasticdl/python/master/k8s_instance_manager.py
@@ -51,7 +51,6 @@ def __init__(
         self._envs = envs
         self._task_d = task_d
         self._next_worker_id = itertools.count().__next__
-        self._next_ps_id = itertools.count().__next__
 
         # Protects followed variables, which are accessed from event_cb.
         self._lock = threading.Lock()
@@ -152,8 +151,8 @@ def start_workers(self):
             self._start_worker(self._next_worker_id())
 
     def start_parameter_servers(self):
-        for _ in range(self._num_ps):
-            self._start_ps(self._next_ps_id())
+        for i in range(self._num_ps):
+            self._start_ps(i)
 
     def _remove_worker(self, worker_id):
         logger.info("Removing worker: %d", worker_id)
@@ -216,8 +215,8 @@ def _event_cb(self, event):
 
         relaunch_worker = False
         relaunch_ps = False
-        worker_id = None
-        ps_id = None
+        worker_id = -1
+        ps_id = -1
         with self._lock:
             if pod_name in self._worker_pod_name_to_id:
                 worker_id = self._worker_pod_name_to_id.get(pod_name)
@@ -244,7 +243,7 @@ def _event_cb(self, event):
                 logger.error("Unknown pod name: %s" % pod_name)
                 return
 
-        if relaunch_worker and worker_id:
+        if relaunch_worker and worker_id >= 0:
             logger.info("Relaunching worker.")
             new_worker_id = self._next_worker_id()
             self._start_worker(new_worker_id)
@@ -254,16 +253,12 @@ def _event_cb(self, event):
                 self._worker_addrs,
                 addr_get_fn=self._k8s_client.get_worker_service_address,
             )
-        elif relaunch_ps and ps_id:
+        elif relaunch_ps:
             logger.info("Relaunching ps.")
-            new_ps_id = self._next_ps_id()
-            self._start_ps(new_ps_id)
-            self._update_addr(
-                ps_id,
-                new_ps_id,
-                self._ps_addrs,
-                addr_get_fn=self._k8s_client.get_ps_service_address,
-            )
+            # Note: the ID and service address for relaunched parameter
+            # server are intentionally left unchanged to support fault
+            # tolerance.
+            self._start_ps(ps_id)
 
     @property
     def ps_addrs(self):


Test patch:
diff --git a/elasticdl/python/tests/k8s_instance_manager_test.py b/elasticdl/python/tests/k8s_instance_manager_test.py
--- a/elasticdl/python/tests/k8s_instance_manager_test.py
+++ b/elasticdl/python/tests/k8s_instance_manager_test.py
@@ -195,7 +195,7 @@ def testRelaunchPsPod(self):
             time.sleep(1)
             with instance_manager._lock:
                 for k in instance_manager._ps_pods_phase:
-                    if k not in range(num_ps, num_ps * 2):
+                    if k not in all_current_ps:
                         found = True
         else:
             self.fail("Failed to find newly launched ps.")