Repository: horovod/horovod

Issue:
Running horovod.spark.run with env=os.environ fails
Example:

    horovod.spark.run(fn, num_proc=2, env=os.environ)

That `env` is an object, not a dictionary. It cannot be pickled:

```
Traceback (most recent call last):
  File "horovod/run/common/util/tiny_shell_exec.py", line 32, in execute
    exit_code = safe_shell_exec.execute(command, env=env, stdout=output, stderr=output)
  File "horovod/run/common/util/safe_shell_exec.py", line 183, in execute
    middleman.start()
  File "multiprocessing/process.py", line 105, in start
    self._popen = self._Popen(self)
  File "multiprocessing/context.py", line 284, in _Popen
    return Popen(process_obj)
  File "multiprocessing/popen_spawn_posix.py", line 32, in __init__
    super().__init__(process_obj)
  File "multiprocessing/popen_fork.py", line 19, in __init__
    self._launch(process_obj)
  File "multiprocessing/popen_spawn_posix.py", line 47, in _launch
    reduction.dump(process_obj, fp)
  File "multiprocessing/reduction.py", line 60, in dump
    ForkingPickler(file, protocol).dump(obj)
AttributeError: Can't pickle local object '_createenviron.<locals>.encode'
```

It works with

    horovod.spark.run(fn, num_proc=2, env=os.environ.copy())

The `run` function needs to copy `env` itself first.


Discussion:

Resolving patch:
diff --git a/horovod/run/mpi_run.py b/horovod/run/mpi_run.py
--- a/horovod/run/mpi_run.py
+++ b/horovod/run/mpi_run.py
@@ -134,6 +134,10 @@ def mpi_run(settings, nics, env, command, stdout=None, stderr=None):
         stderr: Stderr of the mpi process.
                 Only used when settings.run_func_mode is True.
     """
+    if env is not None and not isinstance(env, dict):
+        raise Exception('env argument must be a dict, not {type}: {env}'
+                        .format(type=type(env), env=env))
+
     mpi_impl_flags, impl_binding_args = _get_mpi_implementation_flags(settings.tcp_flag, env=env)
     if mpi_impl_flags is None:
         raise Exception(_MPI_NOT_FOUND_ERROR_MSG)


Test patch:
diff --git a/test/test_run.py b/test/test_run.py
--- a/test/test_run.py
+++ b/test/test_run.py
@@ -34,14 +34,15 @@
 from horovod.run.common.util import codec, config_parser, safe_shell_exec, secret, \
     settings as hvd_settings, timeout
 from horovod.run.common.util.host_hash import _hash, host_hash
+from horovod.run.gloo_run import gloo_run
 from horovod.run.js_run import js_run, generate_jsrun_rankfile
 from horovod.run.mpi_run import _get_mpi_implementation, _get_mpi_implementation_flags,\
     _LARGE_CLUSTER_THRESHOLD as large_cluster_threshold, mpi_available, mpi_run,\
     _OMPI_IMPL, _SMPI_IMPL, _MPICH_IMPL, _UNKNOWN_IMPL, _MISSING_IMPL
-from horovod.run.runner import parse_args, parse_host_files, run_controller, HorovodArgs, _run
+from horovod.run.runner import gloo_built, parse_args, parse_host_files, run_controller, HorovodArgs, _run
 from horovod.run.util.threads import in_thread, on_event
 
-from common import is_built, lsf_and_jsrun, override_args, override_env, temppath, delay, wait
+from common import is_built, lsf_and_jsrun, override_args, temppath, delay, wait
 
 
 class RunTests(unittest.TestCase):
@@ -537,7 +538,7 @@ def test(use_gloo, use_mpi, use_js,
         verbose=0,
         num_hosts=1,
         num_proc=2,
-        hosts='host',
+        hosts='localhost:2',
         run_func_mode=True
     )
 
@@ -563,7 +564,7 @@ def mpi_impl_flags(tcp, env=None):
                 self.assertIsNotNone(mpi_flags)
                 expected_cmd = ('mpirun '
                                 '--allow-run-as-root --tag-output '
-                                '-np 2 -H host '
+                                '-np 2 -H localhost:2 '
                                 '{binding_args} '
                                 '{mpi_flags}       '
                                 'cmd').format(binding_args=' '.join(binding_args), mpi_flags=' '.join(mpi_flags))
@@ -595,7 +596,7 @@ def mpi_impl_flags(tcp, env=None):
                 mpi_flags.append('-mca plm_rsh_num_concurrent {}'.format(settings.num_hosts))
                 expected_cmd = ('mpirun '
                                 '--allow-run-as-root --tag-output '
-                                '-np 2 -H host '
+                                '-np 2 -H localhost:2 '
                                 '{binding_args} '
                                 '{mpi_flags}       '
                                 'cmd').format(binding_args=' '.join(binding_args), mpi_flags=' '.join(mpi_flags))
@@ -671,6 +672,46 @@ def mpi_impl_flags(tcp, env=None):
                 with pytest.raises(RuntimeError, match="^mpirun failed with exit code 1$"):
                     mpi_run(settings, None, {}, cmd)
 
+    """
+    Tests mpi_run with os.environ.
+    """
+    def test_mpi_run_with_os_environ(self):
+        if not mpi_available():
+            self.skipTest("MPI is not available")
+
+        cmd = ['cmd']
+        settings = self.minimal_settings
+
+        def mpi_impl_flags(tcp, env=None):
+            return ["--mock-mpi-impl-flags"], ["--mock-mpi-binding-args"]
+
+        with mock.patch("horovod.run.mpi_run._get_mpi_implementation_flags", side_effect=mpi_impl_flags):
+            with mock.patch("horovod.run.mpi_run.safe_shell_exec.execute", return_value=0):
+                with pytest.raises(Exception, match="^env argument must be a dict, not <class 'os._Environ'>: "):
+                    mpi_run(settings, None, os.environ, cmd)
+
+    """
+    Tests gloo_run with minimal settings.
+    """
+    def test_gloo_run_minimal(self):
+        if not gloo_built:
+            self.skipTest("Gloo is not available")
+
+        cmd = ['whoami']
+        settings = self.minimal_settings
+        gloo_run(settings, ['lo'], {}, '127.0.0.1', cmd)
+
+    """
+    Tests gloo_run with os.environ.
+    """
+    def test_gloo_run_with_os_environ(self):
+        if not gloo_built:
+            self.skipTest("Gloo is not available")
+
+        cmd = ['whoami']
+        settings = self.minimal_settings
+        gloo_run(settings, ['lo'], os.environ, '127.0.0.1', cmd)
+
     def test_horovodrun_hostfile(self):
         with temppath() as host_filename:
             with open(host_filename, 'w+') as fp:
diff --git a/test/test_spark.py b/test/test_spark.py
--- a/test/test_spark.py
+++ b/test/test_spark.py
@@ -473,6 +473,16 @@ def do_test_spark_run_does_not_default_env_to_os_env(self, use_mpi, use_gloo):
             self._do_test_spark_run(env=None, use_mpi=use_mpi, use_gloo=use_gloo,
                                     expected_env=expected_env)
 
+    """
+    Test that horovod.spark.run fails with os.environ as env with mpi.
+    """
+    def test_spark_run_with_os_environ_with_mpi(self):
+        with is_built(gloo_is_built=False, mpi_is_built=True):
+            with spark_session('test_spark_run', cores=2):
+                with pytest.raises(Exception, match="^env argument must be a dict, not <class 'os._Environ'>: "):
+                    horovod.spark.run(fn, num_proc=2, use_mpi=True, use_gloo=False,
+                                      env=os.environ, verbose=2)
+
     """
     Test that horovod.spark.run raises an exception on non-zero exit code of mpi_run using MPI.
     """