Repository: sql-machine-learning/elasticdl

Issue:
Master detects worker pod change and create/re-create RendezvousServer.



Discussion:
We need not to re-create a RendezvousServer and only need to initialize the `host_alloc_plan`.
```python
# Assign host string as worker domain names or IPs.
hosts_str = "172.18.0.22:1,172.18.0.23:1,172.18.0.24:1"
# Set number of workers
num_proc = 3

host_alloc_plan = get_host_plan(hosts_str, num_proc)
rendezvous = RendezvousServer(verbose=True)
global_rendezv_port = rendezvous.start()
rendezvous.init(host_alloc_plan)
```

Resolving patch:
diff --git a/elasticdl/python/master/rendezvous_server.py b/elasticdl/python/master/rendezvous_server.py
new file mode 100644
--- /dev/null
+++ b/elasticdl/python/master/rendezvous_server.py
@@ -0,0 +1,69 @@
+# Copyright 2020 The ElasticDL Authors. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from horovod.runner.common.util.hosts import get_host_assignments, parse_hosts
+from horovod.runner.http.http_server import RendezvousServer
+
+_WORKER_SLOT_NUMBER = 1
+_HOST_SEP = ","
+
+
+class HorovodRendezvousServer(object):
+    def __init__(self, server_host):
+        self._rendezvous_host = server_host
+        self._rendezvous_id = 0
+        self._worker_hosts = []
+        self._rendezvous_server = RendezvousServer(verbose=True)
+        self._rendezvous_port = None
+
+    def start(self):
+        self._rendezvous_port = self._rendezvous_server.start()
+
+    def set_worker_hosts(self, worker_hosts):
+        """
+        Set worker hosts into RendezvousServer
+
+        Args:
+            worker_hosts: String, multiple hosts are separated by ","
+        """
+        if sorted(worker_hosts) == sorted(self._worker_hosts):
+            return
+
+        self._rendezvous_id += 1
+        self._worker_hosts = worker_hosts
+        host_alloc_plan = self._get_host_plan()
+        self._rendezvous_server.init(host_alloc_plan)
+
+    def _get_host_plan(self):
+        hosts = []
+        for host in self._worker_hosts:
+            hosts.append(host + ":" + str(_WORKER_SLOT_NUMBER))
+
+        host_infos = parse_hosts(_HOST_SEP.join(hosts))
+        host_alloc_plan = get_host_assignments(host_infos, len(host_infos))
+        return host_alloc_plan
+
+    def get_rendezvous_host(self):
+        return self._rendezvous_host
+
+    def get_rendezvous_port(self):
+        return self._rendezvous_port
+
+    def get_worker_host_rank(self, host):
+        return self._worker_hosts.index(host)
+
+    def get_size(self):
+        return len(self._worker_hosts)
+
+    def get_rendezvous_id(self):
+        return self._rendezvous_id


Test patch:
diff --git a/elasticdl/python/tests/horovod_rendezvous_test.py b/elasticdl/python/tests/horovod_rendezvous_test.py
new file mode 100644
--- /dev/null
+++ b/elasticdl/python/tests/horovod_rendezvous_test.py
@@ -0,0 +1,59 @@
+# Copyright 2020 The ElasticDL Authors. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from elasticdl.python.master.rendezvous_server import HorovodRendezvousServer
+
+
+class HorovodRendezvousServerTest(unittest.TestCase):
+    def setUp(self):
+        self.rendezvous_server = HorovodRendezvousServer(
+            server_host="127.0.0.1"
+        )
+        self.rendezvous_server.start()
+
+    def test_get_host_plan(self):
+        self.rendezvous_server._worker_hosts = ["127.0.0.2", "127.0.0.3"]
+        host_alloc_plan = self.rendezvous_server._get_host_plan()
+        self.assertEqual(host_alloc_plan[0].hostname, "127.0.0.2")
+        self.assertEqual(host_alloc_plan[0].rank, 0)
+        self.assertEqual(host_alloc_plan[0].size, 2)
+        self.assertEqual(host_alloc_plan[1].hostname, "127.0.0.3")
+        self.assertEqual(host_alloc_plan[1].rank, 1)
+        self.assertEqual(host_alloc_plan[1].size, 2)
+
+    def test_set_worker_hosts(self):
+        worker_hosts = ["127.0.0.2", "127.0.0.3"]
+        self.rendezvous_server.set_worker_hosts(worker_hosts)
+        self.assertEqual(self.rendezvous_server._rendezvous_id, 1)
+
+        new_worker_hosts = ["127.0.0.1", "127.0.0.3"]
+        self.rendezvous_server.set_worker_hosts(new_worker_hosts)
+        self.assertEqual(self.rendezvous_server._rendezvous_id, 2)
+
+    def test_get_attr(self):
+        worker_hosts = ["127.0.0.2", "127.0.0.3"]
+        self.rendezvous_server.set_worker_hosts(worker_hosts)
+        self.assertEqual(
+            self.rendezvous_server.get_rendezvous_host(), "127.0.0.1"
+        )
+        self.assertEqual(
+            self.rendezvous_server.get_worker_host_rank("127.0.0.2"), 0
+        )
+        self.assertEqual(self.rendezvous_server.get_size(), 2)
+        self.assertEqual(self.rendezvous_server.get_rendezvous_id(), 1)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/elasticdl/python/tests/worker_allreduce_strategy_test.py b/elasticdl/python/tests/worker_allreduce_strategy_test.py
deleted file mode 100644
--- a/elasticdl/python/tests/worker_allreduce_strategy_test.py
+++ /dev/null
@@ -1,88 +0,0 @@
-# Copyright 2020 The ElasticDL Authors. All rights reserved.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import unittest
-
-from elasticdl.proto import elasticdl_pb2
-from elasticdl.python.common.args import parse_worker_args
-from elasticdl.python.tests.test_utils import get_mnist_dataset
-from elasticdl.python.worker.worker import Worker
-from elasticdl_client.common.constants import DistributionStrategy
-
-
-class WorkerAllReduceStrategyTest(unittest.TestCase):
-    def setUp(self):
-        self._model_zoo_path = os.path.join(
-            os.path.dirname(os.path.realpath(__file__)), "../../../model_zoo"
-        )
-        self._model_def = "mnist.mnist_functional_api.custom_model"
-        self._batch_size = 16
-        self._test_steps = 10
-        self._workers = []
-        self._create_worker(2)
-
-    def _create_worker(self, worker_num, max_allreduce_retry_num=0):
-        for i in range(worker_num):
-            arguments = [
-                "--worker_id",
-                i,
-                "--num_workers",
-                worker_num,
-                "--job_type",
-                elasticdl_pb2.TRAINING,
-                "--minibatch_size",
-                self._batch_size,
-                "--model_zoo",
-                self._model_zoo_path,
-                "--model_def",
-                self._model_def,
-                "--distribution_strategy",
-                DistributionStrategy.ALLREDUCE,
-            ]
-            args = parse_worker_args(arguments)
-            worker = Worker(
-                args, max_allreduce_retry_num=max_allreduce_retry_num
-            )
-            self._workers.append(worker)
-
-    def test_collect_gradients_with_allreduce_success_case(self):
-        worker = self._workers[0]
-        train_db, _ = get_mnist_dataset(self._batch_size)
-        for step, (x, y) in enumerate(train_db):
-            if step == 0:
-                worker._run_model_call_before_training(x)
-            w_loss, w_grads = worker.training_process_eagerly(x, y)
-            if step == self._test_steps:
-                break
-            self.assertEqual(
-                worker._collect_gradients_with_allreduce_robust(w_grads), True,
-            )
-
-    def test_collect_gradients_with_allreduce_failure_case(self):
-        worker = self._workers[1]
-        train_db, _ = get_mnist_dataset(self._batch_size)
-        for step, (x, y) in enumerate(train_db):
-            if step == 0:
-                worker._run_model_call_before_training(x)
-            if step == self._test_steps:
-                break
-            self.assertEqual(
-                worker._calculate_grads_and_report_with_allreduce(None),
-                False,
-                "Should fail when no data is received",
-            )
-
-
-if __name__ == "__main__":
-    unittest.main()