Repository: horovod/horovod

Issue:
Horovod spark raise error No module named 'pyspark'
**Environment:**
1. Framework: (TensorFlow, Keras, PyTorch, MXNet) Tensorflow
2. Framework version: 
3. Horovod version: Horovod >= 0.19.2
4. MPI version: N/A
5. CUDA version: N/A
6. NCCL version: N/A
7. Python version: 3.7
8. Spark / PySpark version: spark 2.6 (Note: Download spark tarball and deploy spark in a separate directory instead of install pyspark into python site-packages)
9. OS and version: N/A
10. GCC version: N/A

**Bug report:**
Please describe errorneous behavior you're observing and steps to reproduce it.

**Reproduce steps**
```
# Note, don't install pyspark by "pip", if already install pyspark by "pip", uninstall it first.
# Install horovod >= 0.19.2
pip3.7 install --no-cache-dir --force-reinstall horovod==0.19.4

# Uninstall pyspark (if needed)
pip3.7 uninstall pyspark

# Download and untar spark tarball
wget http://apache.mirrors.hoobly.com/spark/spark-3.0.0-preview2/spark-3.0.0-preview2-bin-hadoop2.7.tgz
tar -xf spark-3.0.0-preview2-bin-hadoop2.7.tgz
cd spark-3.0.0-preview2-bin-hadoop2.7

# enter pyspark REPL shell
PYSPARK_PYTHON=python3.7 bin/pyspark
```

Now run test code in pyspark REPL shell:
~~~python
import numpy as np
import horovod.tensorflow.keras as hvd
import horovod.spark
def test_tensorflow():
  hvd.init()
  gathered = hvd.allgather([hvd.rank()])
  assert np.allclose(gathered, list(range(hvd.size())))
  return hvd.rank()
ranks = horovod.spark.run(test_tensorflow, num_proc=2)
~~~

Will get error like:
```
  File "/usr/lib/python3.7/runpy.py", line 183, in _run_module_as_main
    mod_name, mod_spec, code = _get_module_details(mod_name, _Error)
  File "/usr/lib/python3.7/runpy.py", line 109, in _get_module_details
    __import__(pkg_name)
  File "/home/weichen.xu/.local/lib/python3.7/site-packages/horovod/spark/__init__.py", line 18, in <module>
    from .runner import run
  File "/home/weichen.xu/.local/lib/python3.7/site-packages/horovod/spark/runner.py", line 20, in <module>
    import pyspark
ModuleNotFoundError: No module named 'pyspark'
--------------------------------------------------------------------------
ORTE was unable to reliably start one or more daemons.
This usually is caused by:
...
--------------------------------------------------------------------------
Exception in thread Thread-3:
Traceback (most recent call last):
  File "/usr/lib/python3.7/threading.py", line 926, in _bootstrap_inner
    self.run()
  File "/usr/lib/python3.7/threading.py", line 870, in run
    self._target(*self._args, **self._kwargs)
  File "/home/weichen.xu/.local/lib/python3.7/site-packages/horovod/spark/runner.py", line 100, in run_spark
    result = procs.mapPartitionsWithIndex(_make_mapper(driver.addresses(), settings, use_gloo)).collect()
  File "/home/weichen.xu/spark-3.0.0-preview2-bin-hadoop2.7/python/pyspark/rdd.py", line 889, in collect
    sock_info = self.ctx._jvm.PythonRDD.collectAndServe(self._jrdd.rdd())
  File "/home/weichen.xu/spark-3.0.0-preview2-bin-hadoop2.7/python/lib/py4j-0.10.8.1-src.zip/py4j/java_gateway.py", line 1286, in __call__
    answer, self.gateway_client, self.target_id, self.name)
  File "/home/weichen.xu/spark-3.0.0-preview2-bin-hadoop2.7/python/pyspark/sql/utils.py", line 98, in deco
    return f(*a, **kw)
  File "/home/weichen.xu/spark-3.0.0-preview2-bin-hadoop2.7/python/lib/py4j-0.10.8.1-src.zip/py4j/protocol.py", line 328, in get_return_value
    format(target_id, ".", name), value)
py4j.protocol.Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.collectAndServe.
: org.apache.spark.SparkException: Job 0 cancelled part of cancelled job group horovod.spark.run.0
	at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:1989)
	at org.apache.spark.scheduler.DAGScheduler.handleJobCancellation(DAGScheduler.scala:1924)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleJobGroupCancelled$4(DAGScheduler.scala:937)
	at scala.runtime.java8.JFunction1$mcVI$sp.apply(JFunction1$mcVI$sp.java:23)
	at scala.collection.mutable.HashSet.foreach(HashSet.scala:79)
	at org.apache.spark.scheduler.DAGScheduler.handleJobGroupCancelled(DAGScheduler.scala:936)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2175)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2155)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2144)
	at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:758)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2116)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2137)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2156)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2181)
	at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1004)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
	at org.apache.spark.rdd.RDD.withScope(RDD.scala:388)
	at org.apache.spark.rdd.RDD.collect(RDD.scala:1003)
	at org.apache.spark.api.python.PythonRDD$.collectAndServe(PythonRDD.scala:168)
	at org.apache.spark.api.python.PythonRDD.collectAndServe(PythonRDD.scala)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
	at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
	at py4j.Gateway.invoke(Gateway.java:282)
	at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
	at py4j.commands.CallCommand.execute(CallCommand.java:79)
	at py4j.GatewayConnection.run(GatewayConnection.java:238)
	at java.lang.Thread.run(Thread.java:748)


Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/weichen.xu/.local/lib/python3.7/site-packages/horovod/spark/runner.py", line 227, in run
    _launch_job(use_mpi, use_gloo, settings, driver, env, stdout, stderr)
  File "/home/weichen.xu/.local/lib/python3.7/site-packages/horovod/spark/runner.py", line 123, in _launch_job
    settings.verbose)
  File "/home/weichen.xu/.local/lib/python3.7/site-packages/horovod/run/runner.py", line 686, in run_controller
    mpi_run()
  File "/home/weichen.xu/.local/lib/python3.7/site-packages/horovod/spark/runner.py", line 121, in <lambda>
    use_mpi, lambda: mpi_run(settings, nics, driver, env, stdout, stderr),
  File "/home/weichen.xu/.local/lib/python3.7/site-packages/horovod/spark/mpi_run.py", line 54, in mpi_run
    hr_mpi_run(settings, nics, env, command, stdout=stdout, stderr=stderr)
  File "/home/weichen.xu/.local/lib/python3.7/site-packages/horovod/run/mpi_run.py", line 201, in mpi_run
    raise RuntimeError("mpirun failed with exit code {exit_code}".format(exit_code=exit_code))
RuntimeError: mpirun failed with exit code 1
>>> 20/06/18 05:40:55 WARN PythonRunner: Incomplete task 0.0 in stage 0 (TID 0) interrupted: Attempting to kill Python Worker
20/06/18 05:40:55 WARN PythonRunner: Incomplete task 1.0 in stage 0 (TID 1) interrupted: Attempting to kill Python Worker
20/06/18 05:40:56 WARN TaskSetManager: Lost task 1.0 in stage 0.0 (TID 1, ip-10-20-4-87.us-west-2.compute.internal, executor driver): TaskKilled (Stage cancelled)
20/06/18 05:40:56 WARN TaskSetManager: Lost task 0.0 in stage 0.0 (TID 0, ip-10-20-4-87.us-west-2.compute.internal, executor driver): TaskKilled (Stage cancelled)
```

I made some investigation, and already confirmed that this bug was introduced in https://github.com/horovod/horovod/pull/1839



Discussion:
Thanks for reporting @WeichenXu123.  Possibly this is the same error we're seeing in #2029. 

cc @EnricoMi 
@tgaddair  @EnricoMi FYI: Spark deployed in separate directory is a very common case. And the code I provided would be much easier to reproduce the bug than #2029 . :)
Hi @WeichenXu123, thanks for the detailed report. Can you try `PYSPARK_PYTHON` with the absolute path to the python binary, e.g. `PYSPARK_PYTHON=$(which python3.7) bin/pyspark`?
Sorry, that is not the problem. I will investigate this.
As a work around the following should work:

    ranks = horovod.spark.run(test_tensorflow, num_proc=2, env=os.environ)

Let me know if it works for you. I'll try to get this working without the work around.
Alternatively, using Gloo instead of MPI also works for me:

    ranks = horovod.spark.run(test_tensorflow, num_proc=2, use_gloo=True)

Resolving patch:
diff --git a/horovod/run/mpi_run.py b/horovod/run/mpi_run.py
--- a/horovod/run/mpi_run.py
+++ b/horovod/run/mpi_run.py
@@ -187,11 +187,14 @@ def mpi_run(settings, nics, env, command, stdout=None, stderr=None):
     if settings.verbose >= 2:
         print(mpirun_command)
 
-    # we need the driver's PATH in env to run mpirun,
+    # we need the driver's PATH and PYTHONPATH in env to run mpirun,
     # env for mpirun is different to env encoded in mpirun_command
-    if 'PATH' not in env and 'PATH' in os.environ:
-        env = copy.copy(env)  # copy env so we do not leak env modifications
-        env['PATH'] = os.environ['PATH']
+    for var in ['PATH', 'PYTHONPATH']:
+        if var not in env and var in os.environ:
+            # copy env so we do not leak env modifications
+            env = copy.copy(env)
+            # copy var over from os.environ
+            env[var] = os.environ[var]
 
     # Execute the mpirun command.
     if settings.run_func_mode:


Test patch:
diff --git a/test/test_run.py b/test/test_run.py
--- a/test/test_run.py
+++ b/test/test_run.py
@@ -567,6 +567,15 @@ def mpi_impl_flags(tcp, env=None):
                                 '{binding_args} '
                                 '{mpi_flags}       '
                                 'cmd').format(binding_args=' '.join(binding_args), mpi_flags=' '.join(mpi_flags))
+
+                # remove PYTHONPATH from execute's env
+                # we cannot know the exact value of that env variable
+                # so we cannot test it through execute.assert_called_once_with
+                self.assertIn('env', execute.call_args.kwargs)
+                self.assertIn('PYTHONPATH', execute.call_args.kwargs['env'])
+                actual_python_path = execute.call_args.kwargs['env'].pop('PYTHONPATH')
+                self.assertIn(actual_python_path, os.pathsep.join(sys.path))
+
                 expected_env = {'PATH': os.environ.get('PATH')}
                 execute.assert_called_once_with(expected_cmd, env=expected_env, stdout=None, stderr=None)
 
@@ -599,6 +608,15 @@ def mpi_impl_flags(tcp, env=None):
                                 '{binding_args} '
                                 '{mpi_flags}       '
                                 'cmd').format(binding_args=' '.join(binding_args), mpi_flags=' '.join(mpi_flags))
+
+                # remove PYTHONPATH from execute's env
+                # we cannot know the exact value of that env variable
+                # so we cannot test it through execute.assert_called_once_with
+                self.assertIn('env', execute.call_args.kwargs)
+                self.assertIn('PYTHONPATH', execute.call_args.kwargs['env'])
+                actual_python_path = execute.call_args.kwargs['env'].pop('PYTHONPATH')
+                self.assertIn(actual_python_path, os.pathsep.join(sys.path))
+
                 expected_env = {'PATH': os.environ.get('PATH')}
                 execute.assert_called_once_with(expected_cmd, env=expected_env, stdout=None, stderr=None)
 
@@ -653,6 +671,15 @@ def mpi_impl_flags(tcp, env=None):
                                     '-x env1 -x env2 '
                                     '>mpi-extra args go here< '
                                     'cmd arg1 arg2').format(mpi_flags=' '.join(mpi_flags))
+
+                # remove PYTHONPATH from execute's env
+                # we cannot know the exact value of that env variable
+                # so we cannot test it through execute.assert_called_once_with
+                self.assertIn('env', execute.call_args.kwargs)
+                self.assertIn('PYTHONPATH', execute.call_args.kwargs['env'])
+                actual_python_path = execute.call_args.kwargs['env'].pop('PYTHONPATH')
+                self.assertIn(actual_python_path, os.pathsep.join(sys.path))
+
                 expected_env = {'env1': 'val1', 'env2': 'val2', 'PATH': os.environ.get('PATH')}
                 execute.assert_called_once_with(expected_command, env=expected_env, stdout=stdout, stderr=stderr)
 
diff --git a/test/test_spark.py b/test/test_spark.py
--- a/test/test_spark.py
+++ b/test/test_spark.py
@@ -616,6 +616,9 @@ def exception(*args, **argv):
 
         actual_secret = actual_env.pop(secret.HOROVOD_SECRET_KEY, None)
         self.assertEqual(expected_command, actual_command)
+        self.assertIn('PYTHONPATH', actual_env)
+        actual_python_path = actual_env.pop('PYTHONPATH')
+        self.assertIn(actual_python_path, os.pathsep.join(sys.path))
         if env:
             if 'PATH' not in env and 'PATH' in os.environ:
                 env = copy.copy(env)