Repository: sql-machine-learning/elasticdl

Issue:
Support reliable worker pod detection for allreduce
In worker.py, it sleeps for 20 seconds, hoping that is enough for FTLib consensus service to detect worker pod.
This is not reliable and fails often.

```
    if args.distribution_strategy == DistributionStrategy.ALLREDUCE:
        logger.info(
            "Wait for %s seconds for FTLib consensus service to "
            "detect the worker pod" % str(_ALLREDUCE_STRATEGY_WARM_UP_SECS)
        )
        time.sleep(_ALLREDUCE_STRATEGY_WARM_UP_SECS)
```


Discussion:
Currently, workers always crash because of this issue.

```
Traceback (most recent call last):
  File "/usr/lib/python3.6/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/usr/local/lib/python3.6/dist-packages/elasticdl/python/worker/main.py", line 89, in <module>
    main()
  File "/usr/local/lib/python3.6/dist-packages/elasticdl/python/worker/main.py", line 83, in main
    set_parallelism=True,
  File "/usr/local/lib/python3.6/dist-packages/elasticdl/python/worker/worker.py", line 135, in __init__
    self._init_from_args(args)
  File "/usr/local/lib/python3.6/dist-packages/elasticdl/python/worker/worker.py", line 174, in _init_from_args
    if self._distribution_strategy == DistributionStrategy.ALLREDUCE
  File "/usr/local/lib/python3.6/dist-packages/elasticdl/python/collective_ops/communicator.py", line 40, in __init__
    peer_list = list(self._get_peer_set(service_name))
  File "/usr/local/lib/python3.6/dist-packages/elasticdl/python/collective_ops/communicator.py", line 142, in _get_peer_set
    temp_set = socket.getaddrinfo(svc_name, 0, proto=socket.IPPROTO_TCP)
  File "/usr/lib/python3.6/socket.py", line 745, in getaddrinfo
    for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -2] Name or service not known

```

Resolving patch:
diff --git a/elasticdl/python/collective_ops/communicator.py b/elasticdl/python/collective_ops/communicator.py
--- a/elasticdl/python/collective_ops/communicator.py
+++ b/elasticdl/python/collective_ops/communicator.py
@@ -12,6 +12,7 @@
 # limitations under the License.
 
 import socket
+import time
 
 from elasticdl.python.common.constants import CollectiveCommunicatorStatus
 from elasticdl.python.common.log_utils import default_logger as logger
@@ -33,11 +34,39 @@
     "FTLib is not installed. Default to succeeded for testing purposes"
 )
 
+# FTLib consensus service connection timeout in seconds
+_FTLIB_CONSENSUS_CONNECTION_TIMEOUT_SECS = 300
+
 
 class CollectiveCommunicator(object):
     def __init__(self, service_name=None):
         if _FTLIB_INSTALLED:
-            peer_list = list(self._get_peer_set(service_name))
+            connection_try_num = 0
+            while True:
+                try:
+                    peer_list = list(self._get_peer_set(service_name))
+                except Exception:
+                    if (
+                        connection_try_num * 5
+                        > _FTLIB_CONSENSUS_CONNECTION_TIMEOUT_SECS
+                    ):
+                        logger.error(
+                            "Cannot connect to FTLib consensus service in %s "
+                            "seconds",
+                            str(_FTLIB_CONSENSUS_CONNECTION_TIMEOUT_SECS),
+                        )
+                        self._ftlib = None
+                        return
+                    # sleep for 5s and try again
+                    logger.info(
+                        "Cannot connect to FTLib consensus service, "
+                        "trying again."
+                    )
+                    connection_try_num += 1
+                    time.sleep(5)
+                else:
+                    break
+
             self._ftlib = BasicFTLib(
                 consensus="gossip",
                 commlib="pytorch",
diff --git a/elasticdl/python/common/model_handler.py b/elasticdl/python/common/model_handler.py
--- a/elasticdl/python/common/model_handler.py
+++ b/elasticdl/python/common/model_handler.py
@@ -193,11 +193,6 @@ def get_model_handler(
         """
         if distribution_strategy == DistributionStrategy.PARAMETER_SERVER:
             return ParameterServerModelHandler(checkpoint_dir=checkpoint_dir)
-        elif distribution_strategy == DistributionStrategy.ALLREDUCE:
-            logger.warning(
-                "Allreduce distribution strategy is not supported yet. "
-                "Switching to use the default distribution strategy."
-            )
         return DefaultModelHandler()
 
 
diff --git a/elasticdl/python/worker/main.py b/elasticdl/python/worker/main.py
--- a/elasticdl/python/worker/main.py
+++ b/elasticdl/python/worker/main.py
@@ -11,21 +11,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import time
-
 import grpc
 
 from elasticdl.python.common import log_utils
 from elasticdl.python.common.args import parse_worker_args
 from elasticdl.python.common.grpc_utils import build_channel
 from elasticdl.python.worker.worker import Worker
-from elasticdl_client.common.constants import DistributionStrategy
 
 CONNECT_PS_MAX_RETRIES = 3
 CONNECT_PS_TIMEOUT = 300
-# The number of seconds we wait for allreduce strategy's
-# FTLib consensus service to detect the worker pod
-_ALLREDUCE_STRATEGY_WARM_UP_SECS = 20
 
 
 def main():
@@ -69,13 +63,6 @@ def main():
                     % addr.split(".")[0]
                 )
 
-    if args.distribution_strategy == DistributionStrategy.ALLREDUCE:
-        logger.info(
-            "Wait for %s seconds for FTLib consensus service to "
-            "detect the worker pod" % str(_ALLREDUCE_STRATEGY_WARM_UP_SECS)
-        )
-        time.sleep(_ALLREDUCE_STRATEGY_WARM_UP_SECS)
-
     worker = Worker(
         args,
         channel=master_channel,


Test patch: