# Buggy code
def _run_static(args):
    all_host_names, _ = parse_hosts_and_slots(args.hosts)

    nics_set = set(args.nics.split(",")) if args.nics else None

    # horovodrun has to finish all the checks before this timeout runs out.
    if args.start_timeout:
        start_timeout = args.start_timeout
    else:
        # Lookup default timeout from the environment variable.
        start_timeout = int(os.getenv("HOROVOD_START_TIMEOUT", "30"))

    tmout = timeout.Timeout(
        start_timeout,
        message="Timed out waiting for {activity}. Please "
        "check connectivity between servers. You "
        "may need to increase the --start-timeout "
        "parameter if you have too many servers.",
    )
    settings = hvd_settings.Settings(
        verbose=2 if args.verbose else 0,
        ssh_port=args.ssh_port,
        extra_mpi_args=args.mpi_args,
        tcp_flag=args.tcp_flag,
        binding_args=args.binding_args,
        key=secret.make_secret_key(),
        start_timeout=tmout,
        num_proc=args.np,
        hosts=args.hosts,
        num_hosts=len(all_host_names),
        output_filename=args.output_filename,
        run_func_mode=args.run_func is not None,
        nics=nics_set,
    )

    # This cache stores the results of checks performed by horovod
    # during the initialization step. It can be disabled by setting
    # --disable-cache flag.
    fn_cache = None
    if not args.disable_cache:
        params = ""
        if args.np:
            params += str(args.np) + " "
        if args.hosts:
            params += str(args.hosts) + " "
        if args.ssh_port:
            params += str(args.ssh_port)
        parameters_hash = hashlib.md5(params.encode("utf-8")).hexdigest()
        fn_cache = cache.Cache(
            CACHE_FOLDER, CACHE_STALENESS_THRESHOLD_MINUTES, parameters_hash
        )

    if settings.verbose >= 2:
        print("Filtering local host names.")
    remote_host_names = network.filter_local_addresses(all_host_names)
    if settings.verbose >= 2:
        print("Remote host found: " + " ".join(remote_host_names))

    if len(remote_host_names) > 0:
        if settings.verbose >= 2:
            print("Checking ssh on all remote hosts.")
        # Check if we can ssh into all remote hosts successfully.
        _check_all_hosts_ssh_successful(
            remote_host_names, args.ssh_port, fn_cache=fn_cache
        )
        if settings.verbose >= 2:
            print("SSH was successful into all the remote hosts.")

    nics = driver_service.get_common_interfaces(
        settings, all_host_names, remote_host_names, fn_cache
    )

    if args.run_func:
        # get the driver IPv4 address
        driver_ip = network.get_driver_ip(nics)
        run_func_server = KVStoreServer(verbose=settings.verbose)
        run_func_server_port = run_func_server.start_server()
        put_data_into_kvstore(
            driver_ip, run_func_server_port, "runfunc", "func", args.run_func
        )

        command = [
            sys.executable,
            "-m",
            "horovod.run.run_task",
            str(driver_ip),
            str(run_func_server_port),
        ]

        try:
            _launch_job(args, settings, nics, command)
            results = [None] * args.np
            # TODO: make it parallel to improve performance
            for i in range(args.np):
                results[i] = read_data_from_kvstore(
                    driver_ip, run_func_server_port, "runfunc_result", str(i)
                )
            return results
        finally:
            run_func_server.shutdown_server()
    else:
        command = args.command
        _launch_job(args, settings, nics, command)
        return None


# Traceback (struct.error)
root@ip-172-31-37-52:/# horovodrun -np 2 -H localhost:1,172.31.35.37:1 -p 12345 --verbose ls
Filtering local host names.
Remote host found: 172.31.35.37
Checking ssh on all remote hosts.
SSH was successful into all the remote hosts.
Testing interfaces on all the hosts.
Launched horovod server.
Attempted to launch horovod task servers.
Waiting for the hosts to acknowledge.
----------------------------------------
----------------------------------------
Exception happened during processing of request from ('127.0.0.1', 37436)
Exception happened during processing of request from ('172.31.37.52', 52632)
----------------------------------------
Exception happened during processing of request from ('172.31.37.52', 52636)
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/socketserver.py", line 654, in process_request_thread
self.finish_request(request, client_address)
File "/opt/conda/lib/python3.6/socketserver.py", line 364, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/opt/conda/lib/python3.6/socketserver.py", line 724, in __init__
self.handle()
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 105, in handle
req = server._wire.read(self.rfile)
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 79, in read
message_len = struct.unpack('i', rfile.read(4))[0]
Traceback (most recent call last):
struct.error: unpack requires a buffer of 4 bytes
----------------------------------------
File "/opt/conda/lib/python3.6/socketserver.py", line 654, in process_request_thread
self.finish_request(request, client_address)
File "/opt/conda/lib/python3.6/socketserver.py", line 364, in finish_request
self.RequestHandlerClass(request, client_address, self)
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/socketserver.py", line 724, in __init__
self.handle()
File "/opt/conda/lib/python3.6/socketserver.py", line 654, in process_request_thread
self.finish_request(request, client_address)
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 105, in handle
req = server._wire.read(self.rfile)
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 79, in read
message_len = struct.unpack('i', rfile.read(4))[0]
struct.error: unpack requires a buffer of 4 bytes
----------------------------------------
File "/opt/conda/lib/python3.6/socketserver.py", line 364, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/opt/conda/lib/python3.6/socketserver.py", line 724, in __init__
self.handle()
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 105, in handle
req = server._wire.read(self.rfile)
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 79, in read
message_len = struct.unpack('i', rfile.read(4))[0]
struct.error: unpack requires a buffer of 4 bytes
----------------------------------------
----------------------------------------
----------------------------------------
Exception happened during processing of request from ('172.31.37.52', 55358)
----------------------------------------
Exception happened during processing of request from ('172.31.37.52', 52642)
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/socketserver.py", line 654, in process_request_thread
self.finish_request(request, client_address)
File "/opt/conda/lib/python3.6/socketserver.py", line 364, in finish_request
self.RequestHandlerClass(request, client_address, self)
Exception happened during processing of request from ('127.0.0.1', 37442)
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/socketserver.py", line 654, in process_request_thread
self.finish_request(request, client_address)
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/socketserver.py", line 654, in process_request_thread
self.finish_request(request, client_address)
File "/opt/conda/lib/python3.6/socketserver.py", line 364, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/opt/conda/lib/python3.6/socketserver.py", line 724, in __init__
self.handle()
File "/opt/conda/lib/python3.6/socketserver.py", line 724, in __init__
self.handle()
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 105, in handle
req = server._wire.read(self.rfile)
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 79, in read
message_len = struct.unpack('i', rfile.read(4))[0]
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 105, in handle
req = server._wire.read(self.rfile)
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 79, in read
message_len = struct.unpack('i', rfile.read(4))[0]
File "/opt/conda/lib/python3.6/socketserver.py", line 364, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/opt/conda/lib/python3.6/socketserver.py", line 724, in __init__
self.handle()
struct.error: unpack requires a buffer of 4 bytes
struct.error: unpack requires a buffer of 4 bytes
----------------------------------------
----------------------------------------
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 105, in handle
req = server._wire.read(self.rfile)
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 79, in read
message_len = struct.unpack('i', rfile.read(4))[0]
struct.error: unpack requires a buffer of 4 bytes
----------------------------------------
Exception happened during processing of request from ('172.31.37.52', 55364)
----------------------------------------
Exception happened during processing of request from ('127.0.0.1', 37448)
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/socketserver.py", line 654, in process_request_thread
self.finish_request(request, client_address)
File "/opt/conda/lib/python3.6/socketserver.py", line 364, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/opt/conda/lib/python3.6/socketserver.py", line 724, in __init__
self.handle()
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 105, in handle
req = server._wire.read(self.rfile)
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 79, in read
message_len = struct.unpack('i', rfile.read(4))[0]
struct.error: unpack requires a buffer of 4 bytes
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/socketserver.py", line 654, in process_request_thread
self.finish_request(request, client_address)
----------------------------------------
----------------------------------------
----------------------------------------
Exception happened during processing of request from ('172.31.37.52', 55368)
File "/opt/conda/lib/python3.6/socketserver.py", line 364, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/opt/conda/lib/python3.6/socketserver.py", line 724, in __init__
self.handle()
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 105, in handle
req = server._wire.read(self.rfile)
File "/opt/conda/lib/python3.6/socketserver.py", line 654, in process_request_thread
self.finish_request(request, client_address)
File "/opt/conda/lib/python3.6/socketserver.py", line 364, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/opt/conda/lib/python3.6/socketserver.py", line 724, in __init__
self.handle()
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 105, in handle
req = server._wire.read(self.rfile)
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 79, in read
message_len = struct.unpack('i', rfile.read(4))[0]
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 79, in read
message_len = struct.unpack('i', rfile.read(4))[0]
struct.error: unpack requires a buffer of 4 bytes
----------------------------------------
struct.error: unpack requires a buffer of 4 bytes
----------------------------------------
----------------------------------------
Exception happened during processing of request from ('172.31.35.37', 39790)
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/socketserver.py", line 654, in process_request_thread
self.finish_request(request, client_address)
File "/opt/conda/lib/python3.6/socketserver.py", line 364, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/opt/conda/lib/python3.6/socketserver.py", line 724, in __init__
self.handle()
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 105, in handle
req = server._wire.read(self.rfile)
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 79, in read
message_len = struct.unpack('i', rfile.read(4))[0]
struct.error: unpack requires a buffer of 4 bytes
----------------------------------------
----------------------------------------
Exception happened during processing of request from ('172.31.35.37', 39796)
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/socketserver.py", line 654, in process_request_thread
self.finish_request(request, client_address)
File "/opt/conda/lib/python3.6/socketserver.py", line 364, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/opt/conda/lib/python3.6/socketserver.py", line 724, in __init__
self.handle()
----------------------------------------
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 105, in handle
req = server._wire.read(self.rfile)
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 79, in read
message_len = struct.unpack('i', rfile.read(4))[0]
struct.error: unpack requires a buffer of 4 bytes
----------------------------------------
Exception happened during processing of request from ('172.31.35.37', 39798)
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/socketserver.py", line 654, in process_request_thread
self.finish_request(request, client_address)
File "/opt/conda/lib/python3.6/socketserver.py", line 364, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/opt/conda/lib/python3.6/socketserver.py", line 724, in __init__
self.handle()
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 105, in handle
req = server._wire.read(self.rfile)
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 79, in read
message_len = struct.unpack('i', rfile.read(4))[0]
struct.error: unpack requires a buffer of 4 bytes
----------------------------------------
Launching horovod task function was not successful:
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "/opt/conda/lib/python3.6/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/opt/conda/lib/python3.6/site-packages/horovod/run/task_fn.py", line 67, in <module>
_task_fn(index, driver_addresses, settings)
File "/opt/conda/lib/python3.6/site-packages/horovod/run/task_fn.py", line 27, in _task_fn
driver_addresses, settings.key, settings.verbose)
File "/opt/conda/lib/python3.6/site-packages/horovod/run/driver/driver_service.py", line 44, in __init__
match_intf=match_intf)
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/service/driver_service.py", line 159, in __init__
match_intf=match_intf)
File "/opt/conda/lib/python3.6/site-packages/horovod/run/common/util/network.py", line 172, in __init__
'Linux.'.format(service_name=service_name, addresses=addresses))
horovod.run.common.util.network.NoValidAddressesFound: Horovod was unable to connect to horovod driver service on any of the following addresses: {'lo': [('127.0.0.1', 4548)], 'ens3': [('172.31.37.52', 4548)], 'docker0': [('172.17.0.1', 4548)]}.

One possible cause of this problem is that horovod currently requires every host to have at least one routable network interface with the same name across all of the hosts. You can run "ifconfig -a" on every host and check for the common routable interface. To fix the problem, you can rename interfaces on Linux.

# Fixed code
def _run_static(args):
    all_host_names, _ = parse_hosts_and_slots(args.hosts)

    nics_set = set(args.nics.split(",")) if args.nics else None

    # horovodrun has to finish all the checks before this timeout runs out.
    if args.start_timeout:
        start_timeout = args.start_timeout
    else:
        # Lookup default timeout from the environment variable.
        start_timeout = int(os.getenv("HOROVOD_START_TIMEOUT", "30"))

    tmout = timeout.Timeout(
        start_timeout,
        message="Timed out waiting for {activity}. Please "
        "check connectivity between servers. You "
        "may need to increase the --start-timeout "
        "parameter if you have too many servers.",
    )
    settings = hvd_settings.Settings(
        verbose=2 if args.verbose else 0,
        ssh_port=args.ssh_port,
        extra_mpi_args=args.mpi_args,
        tcp_flag=args.tcp_flag,
        binding_args=args.binding_args,
        key=secret.make_secret_key(),
        start_timeout=tmout,
        num_proc=args.np,
        hosts=args.hosts,
        num_hosts=len(all_host_names),
        output_filename=args.output_filename,
        run_func_mode=args.run_func is not None,
        nics=nics_set,
    )

    # This cache stores the results of checks performed by horovod
    # during the initialization step. It can be disabled by setting
    # --disable-cache flag.
    fn_cache = None
    if not args.disable_cache:
        params = ""
        if args.np:
            params += str(args.np) + " "
        if args.hosts:
            params += str(args.hosts) + " "
        if args.ssh_port:
            params += str(args.ssh_port)
        parameters_hash = hashlib.md5(params.encode("utf-8")).hexdigest()
        fn_cache = cache.Cache(
            CACHE_FOLDER, CACHE_STALENESS_THRESHOLD_MINUTES, parameters_hash
        )

    if settings.verbose >= 2:
        print("Filtering local host names.")
    remote_host_names = network.filter_local_addresses(all_host_names)
    if settings.verbose >= 2:
        print("Remote host found: " + " ".join(remote_host_names))

    if len(remote_host_names) > 0:
        if settings.verbose >= 2:
            print("Checking ssh on all remote hosts.")
        # Check if we can ssh into all remote hosts successfully.
        if not _check_all_hosts_ssh_successful(
            remote_host_names, args.ssh_port, fn_cache=fn_cache
        ):
            raise RuntimeError("could not connect to some hosts via ssh")
        if settings.verbose >= 2:
            print("SSH was successful into all the remote hosts.")

    nics = driver_service.get_common_interfaces(
        settings, all_host_names, remote_host_names, fn_cache
    )

    if args.run_func:
        # get the driver IPv4 address
        driver_ip = network.get_driver_ip(nics)
        run_func_server = KVStoreServer(verbose=settings.verbose)
        run_func_server_port = run_func_server.start_server()
        put_data_into_kvstore(
            driver_ip, run_func_server_port, "runfunc", "func", args.run_func
        )

        command = [
            sys.executable,
            "-m",
            "horovod.run.run_task",
            str(driver_ip),
            str(run_func_server_port),
        ]

        try:
            _launch_job(args, settings, nics, command)
            results = [None] * args.np
            # TODO: make it parallel to improve performance
            for i in range(args.np):
                results[i] = read_data_from_kvstore(
                    driver_ip, run_func_server_port, "runfunc_result", str(i)
                )
            return results
        finally:
            run_func_server.shutdown_server()
    else:
        command = args.command
        _launch_job(args, settings, nics, command)
        return None