Repository: horovod/horovod

Issue:
horovod.common.exceptions.HorovodInternalError: Broadcast is not supported with Join at this time.
**Environment:**
1. Framework: Pytorch (Lightning)
2. Framework version: 1.9.0+cu102 (and 1.4.2 for pytorch lightning)
3. Horovod version: 0.22.1
4. MPI version: 2.1.1
5. CUDA version: 10.2
6. NCCL version: 2708
7. Python version: 3.9.6
8. Spark / PySpark version: -
9. Ray version: -
10. OS and version: Ubuntu 18.04
11. GCC version:  gcc (Ubuntu 7.5.0-3ubuntu1~18.04) 7.5.0
12. CMake version: cmake version 3.10.2


**Bug report:**
When running my training loop using horovod as an accelerator in pytorch lightning I encounter the following error after the 2nd epoch in all of my 8 workers:
```
[0]<stderr>:  File "/usr/local/lib/python3.9/dist-packages/pytorch_lightning/callbacks/model_checkpoint.py", line 816, in file_exists
[0]<stderr>:    return trainer.training_type_plugin.broadcast(exists)
[0]<stderr>:           │       │                              └ False
[0]<stderr>:           │       └ <property object at 0x7fa7182721d0>
[0]<stderr>:           └ <pytorch_lightning.trainer.trainer.Trainer object at 0x7fa708526fa0>
[0]<stderr>:
[0]<stderr>:  File "/usr/local/lib/python3.9/dist-packages/pytorch_lightning/plugins/training_type/horovod.py", line 129, in broadcast
[0]<stderr>:    obj = hvd.broadcast_object(obj, src)
[0]<stderr>:          │   │                │    └ 0
[0]<stderr>:          │   │                └ False
[0]<stderr>:          │   └ <function broadcast_object at 0x7fa72eb0c4c0>
[0]<stderr>:          └ <module 'horovod.torch' from '/usr/local/lib/python3.9/dist-packages/horovod/torch/__init__.py'>
[0]<stderr>:
[0]<stderr>:  File "/usr/local/lib/python3.9/dist-packages/horovod/torch/functions.py", line 218, in broadcast_object
[0]<stderr>:    broadcast_(sz, root_rank, name + '.sz')
[0]<stderr>:    │          │   │          └ 'bool'
[0]<stderr>:    │          │   └ 0
[0]<stderr>:    │          └ tensor([4], dtype=torch.int32)
[0]<stderr>:    └ <function broadcast_ at 0x7fa72eb0a040>
[0]<stderr>:
[0]<stderr>:  File "/usr/local/lib/python3.9/dist-packages/horovod/torch/mpi_ops.py", line 741, in broadcast_
[0]<stderr>:    return synchronize(handle)
[0]<stderr>:           │           └ 69713
[0]<stderr>:           └ <function synchronize at 0x7fa72eb0a4c0>
[0]<stderr>:
[0]<stderr>:  File "/usr/local/lib/python3.9/dist-packages/horovod/torch/mpi_ops.py", line 882, in synchronize
[0]<stderr>:    raise HorovodInternalError(e)
[0]<stderr>:          └ <class 'horovod.common.exceptions.HorovodInternalError'>

[0]<stderr>:horovod.common.exceptions.HorovodInternalError: Broadcast is not supported with Join at this time.
```

I'm not quite sure how to avoid the broadcast in the Join since this is all code internal to pytorch lightning. Do you have any idea what might be causing this? If you think this is an issue with pytorch lightning I'm happy to open an issue there. 
Thanks a lot!



Discussion:
Looks like the join() op is used as a barrier in Lightning, it shouldn't be used like that. I think we need to introduce a barrier op in horovod.torch and change Lightning to use that. I can take a look.

Resolving patch:
diff --git a/horovod/common/basics.py b/horovod/common/basics.py
--- a/horovod/common/basics.py
+++ b/horovod/common/basics.py
@@ -150,7 +150,8 @@ def shutdown(self):
 
     def is_initialized(self):
         """Returns True if Horovod is initialized"""
-        return self.MPI_LIB_CTYPES.horovod_is_initialized()
+        is_initialized = self.MPI_LIB_CTYPES.horovod_is_initialized()
+        return bool(is_initialized)
 
     def start_timeline(self, file_path, mark_cycles=False):
         """Creates a timeline file at `file_path` and begins recording.
diff --git a/horovod/torch/__init__.py b/horovod/torch/__init__.py
--- a/horovod/torch/__init__.py
+++ b/horovod/torch/__init__.py
@@ -42,6 +42,7 @@
     from horovod.torch.mpi_ops import broadcast, broadcast_async, broadcast_, broadcast_async_
     from horovod.torch.mpi_ops import alltoall, alltoall_async
     from horovod.torch.mpi_ops import join
+    from horovod.torch.mpi_ops import barrier
     from horovod.torch.mpi_ops import poll, synchronize
     from horovod.torch.mpi_ops import init, shutdown
     from horovod.torch.mpi_ops import is_initialized, start_timeline, stop_timeline
diff --git a/horovod/torch/mpi_ops.py b/horovod/torch/mpi_ops.py
--- a/horovod/torch/mpi_ops.py
+++ b/horovod/torch/mpi_ops.py
@@ -970,3 +970,23 @@ def join(device=-1) -> int:
     _handle_map[handle] = (None, output)
 
     return synchronize(handle).item()
+
+def barrier(process_set=global_process_set):
+    """
+    A function that acts as a simple sychronization point for ranks specified
+    in the given process group(default to global group). Ranks that reach
+    this function call will stall until all other ranks have reached.
+
+    Arguments:
+        process_set: Process set object to limit this operation to a subset of
+                     Horovod processes. Default is the global process set.
+    """
+
+    try:
+        handle = mpi_lib.horovod_torch_barrier(process_set.process_set_id)
+    except RuntimeError as e:
+        raise HorovodInternalError(e)
+
+    _handle_map[handle] = (None, None)
+
+    synchronize(handle)


Test patch:
diff --git a/test/parallel/test_torch.py b/test/parallel/test_torch.py
--- a/test/parallel/test_torch.py
+++ b/test/parallel/test_torch.py
@@ -28,6 +28,7 @@
 import json
 
 from collections.abc import Iterable
+from datetime import datetime
 
 import numpy as np
 import pytest
@@ -3179,7 +3180,74 @@ def create_model(opt_class, opt_params, process_set):
         hvd.remove_process_set(odd_set)
         hvd.remove_process_set(even_set)
 
+    def test_process_set_barrier_op(self):
+        """Test that process set barrier stalls all ranks in that process set"""
+        hvd.init()
+
+        # No need to test if only one rank is available
+        if hvd.size() == 1:
+            self.skipTest("Number of ranks is 1. Skipping test.")
+
+        even_ranks = [rk for rk in range(0, hvd.size()) if rk % 2 == 0]
+        odd_ranks = [rk for rk in range(0, hvd.size()) if rk % 2 == 1]
+        even_set = hvd.add_process_set(even_ranks)
+        odd_set = hvd.add_process_set(odd_ranks)
+
+        # Make sure all ranks are initialized
+        i = 0
+        while hvd.allreduce_(torch.IntTensor([int(hvd.is_initialized())]), None, 'is_initialized{}'.format(i), hvd.Sum) != hvd.size():
+            i+=1
+            continue
 
+        even_barrier_time = 0
+        odd_barrier_time = 0
+        even_barrier_time_start = datetime.now()
+        odd_barrier_time_start = datetime.now()
+
+        if hvd.rank() in even_ranks:
+            # rank 0 sleeps for 5 seconds
+            if hvd.rank() == 0:
+                time.sleep(5)
+            hvd.barrier(even_set)
+            # barrier time should be at least 5 seconds for all even ranks
+            even_barrier_time_end = datetime.now()
+            even_barrier_time = (even_barrier_time_end - even_barrier_time_start).total_seconds()
+            self.assertTrue(even_barrier_time >= 5)
+        # No stall time for odd ranks
+        elif hvd.rank() in odd_ranks:
+            hvd.barrier(odd_set)
+            odd_barrier_time_end = datetime.now()
+            odd_barrier_time = (odd_barrier_time_end - odd_barrier_time_start).total_seconds()
+            self.assertTrue(odd_barrier_time <= 1)
+
+        hvd.barrier()
+
+    def test_global_barrier_op(self):
+        """Test that global barrier stalls all ranks"""
+        hvd.init()
+
+        # No need to test if only one rank is available
+        if hvd.size() == 1:
+            self.skipTest("Number of ranks is 1. Skipping test.")
+
+        # Make sure all ranks are initialized
+        i = 0
+        while hvd.allreduce_(torch.IntTensor([int(hvd.is_initialized())]), None, 'is_initialized{}'.format(i), hvd.Sum) != hvd.size():
+            i+=1
+            continue
+
+        # Sleep rank 0 for 5 seconds, all the other ranks will arrive barrier right away.
+        barrier_time = 0
+        barrier_time_start = datetime.now()
+        if hvd.rank() == 0:
+            time.sleep(5)
+        hvd.barrier()
+
+        # barrier time should be at least 5 seconds for all ranks
+        barrier_time_end = datetime.now()
+        barrier_time = (barrier_time_end - barrier_time_start).total_seconds()
+
+        self.assertTrue(barrier_time >= 5)
 
 if __name__ == "__main__":
    unittest.main()