Repository: horovod/horovod

Issue:
【Bug】Horovod pytorch function hvd.join not return last joined rank id
## Environment:

Framework: Pytorch
Framework version: 1.8.1+cu111
Horovod version: 0.22.0
MPI version: 
CUDA version: 11.1
NCCL version: 2.9.6
Python version: 3.6
Spark / PySpark version:
Ray version:
OS and version: debian buster/sid
GCC version: 5.4.0
CMake version:

## Checklist:

Did you search issues to find if somebody asked this question before? yes
If your question is about hang, did you read this doc?
If your question is about docker, did you read this doc?
Did you check if you question is answered in the troubleshooting guide? yes

## Bug report:

As the document https://horovod.readthedocs.io/en/stable/api.html where:
horovod.torch.join() function will block until all ranks join, and the function will return the Id of the rank that joined last.
However, with the test code below, it seems that the join() function return something out of expected.

```
from distutils.version import LooseVersion
import itertools
import torch
import horovod.torch as hvd
_1_5_api = LooseVersion(torch.__version__) >= LooseVersion('1.5.0')

def cast_and_place(tensor, dtype):
        if dtype.is_cuda:
            return tensor.cuda(hvd.local_rank()).type(dtype)
        return tensor.type(dtype)

def test_horovod_join_allreduce():
        """Test Join op with allreduce."""
        hvd.init()
        rank = hvd.rank()
        size = hvd.size()
        dtypes = [torch.cuda.IntTensor, torch.cuda.LongTensor, torch.cuda.FloatTensor, torch.cuda.DoubleTensor, torch.cuda.HalfTensor]
        integral_types = [torch.IntTensor, torch.LongTensor, torch.cuda.IntTensor, torch.cuda.LongTensor]
        dims = [1, 2, 3]
        first_join_ranks = [0, 1]
        cachings = [False, True]
        for dtype, dim, first_join_rank, caching in itertools.product(dtypes, dims, first_join_ranks, cachings):
            torch.manual_seed(1234)

            def div(t, s):
                if _1_5_api and dtype in integral_types:
                    return t.floor_divide(s)
                return t / s

            # Use two tensors to test fusion
            tensor_a = torch.FloatTensor(*([5] * dim)).random_(-100, 100)
            tensor_a = cast_and_place(tensor_a, dtype)
            tensor_b = torch.FloatTensor(*([17] * dim)).random_(-100, 100)
            tensor_b = cast_and_place(tensor_b, dtype)

            if caching:
                handle_a = hvd.allreduce_async(tensor_a, name="tensor_a", average=True)
                handle_b = hvd.allreduce_async(tensor_b, name="tensor_b", average=True)
                averaged_a = hvd.synchronize(handle_a)
                averaged_b = hvd.synchronize(handle_b)

            if rank == first_join_rank:
                if dtype.is_cuda:
                    ret = hvd.join(hvd.local_rank())
                else:
                    ret = hvd.join()
                print("Last join rank: {}".format(ret))
            else:
                handle_a = hvd.allreduce_async(tensor_a, name="tensor_a", average=True)
                handle_b = hvd.allreduce_async(tensor_b, name="tensor_b", average=True)
                averaged_a = hvd.synchronize(handle_a)
                averaged_b = hvd.synchronize(handle_b)
                if dtype.is_cuda:
                    ret = hvd.join(hvd.local_rank())
                else:
                    ret = hvd.join()
                print("Last join rank: {}".format(ret))

                # Threshold for floating point equality depends on number of
                # ranks, since we're comparing against precise multiplication.
                if size <= 3 or dtype in integral_types:
                    threshold = 0
                elif size < 10:
                    threshold = 1e-4
                elif size < 15:
                    threshold = 5e-4
                else:
                    break
                assert torch.allclose(averaged_a, div(tensor_a * (size - 1), size), threshold), \
                    'hvd.join with hvd.allreduce produces incorrect results'
                assert torch.allclose(averaged_b, div(tensor_b * (size - 1), size), threshold), \
                    'hvd.join with hvd.allreduce produces incorrect results'

test_horovod_join_allreduce()
```
Horovod version: v0.22.0

Command:
```
HOROVOD_CACHE_CAPACITY=0 horovodrun -np 8 -H localhost:8 python3 test.py
```

Output of the test code:
```
[1,0]<stdout>:Last join rank: 7
[1,3]<stdout>:Last join rank: 11
[1,5]<stdout>:Last join rank: 11
[1,4]<stdout>:Last join rank: 11
[1,1]<stdout>:Last join rank: 9
[1,7]<stdout>:Last join rank: 11
[1,6]<stdout>:Last join rank: 11
[1,2]<stdout>:Last join rank: 11
[1,0]<stdout>:Last join rank: 12
[1,4]<stdout>:Last join rank: 16
[1,1]<stdout>:Last join rank: 12
[1,3]<stdout>:Last join rank: 16
[1,5]<stdout>:Last join rank: 16
[1,7]<stdout>:Last join rank: 16
[1,6]<stdout>:Last join rank: 16
[1,2]<stdout>:Last join rank: 16
[1,6]<stdout>:Last join rank: 19
[1,0]<stdout>:Last join rank: 13
[1,3]<stdout>:Last join rank: 19
[1,5]<stdout>:Last join rank: 19
[1,7]<stdout>:Last join rank: 19
[1,2]<stdout>:Last join rank: 19
[1,4]<stdout>:Last join rank: 19
[1,1]<stdout>:Last join rank: 15
[1,6]<stdout>:Last join rank: 24
[1,2]<stdout>:Last join rank: 24
[1,4]<stdout>:Last join rank: 24
[1,0]<stdout>:Last join rank: 16
[1,3]<stdout>:Last join rank: 24
[1,5]<stdout>:Last join rank: 24
[1,1]<stdout>:Last join rank: 20
[1,7]<stdout>:Last join rank: 24
[1,1]<stdout>:Last join rank: 21
[1,5]<stdout>:Last join rank: 27
[1,6]<stdout>:Last join rank: 27
[1,2]<stdout>:Last join rank: 27
[1,4]<stdout>:Last join rank: 27
[1,0]<stdout>:Last join rank: 19
[1,3]<stdout>:Last join rank: 27
[1,7]<stdout>:Last join rank: 27
[1,3]<stdout>:Last join rank: 32
[1,1]<stdout>:Last join rank: 24
[1,5]<stdout>:Last join rank: 32
[1,6]<stdout>:Last join rank: 32
```


Discussion:
This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.

I looked into this a bit out of curiosity, but just by reading code I could not quickly identify where in the code base the output of the Join op is actually set to a value. [It's initially set to an empty (uninitialized) tensor here:] https://github.com/horovod/horovod/blob/6c6eee473714d4761db31bd769c24a852d503d7e/horovod/torch/mpi_ops_v2.cc#L595

@kit1980, if you're still involved with the project, would you know what could be going on here?
@maxhgerlach yeah, as I mentioned in https://github.com/horovod/horovod/issues/832#issuecomment-702858430, I forgot to properly implement this because in my use cases I always knew the rank that will join last.

I don't really use horovod anymore.
Thanks for the explanation, @kit1980! I'll take a stab at this one then.

Resolving patch:
diff --git a/horovod/tensorflow/mpi_ops.py b/horovod/tensorflow/mpi_ops.py
--- a/horovod/tensorflow/mpi_ops.py
+++ b/horovod/tensorflow/mpi_ops.py
@@ -355,6 +355,14 @@ def _alltoall_grad(op, grad_wrt_output, grad_wrt_received_splits):
     return [grad_wrt_tensor, grad_wrt_splits]
 
 def join():
+    """An op to indicate that the rank finished processing data.
+
+    All ranks that did not call join() continue to process allreduce
+    operations. This op is not done before all ranks have joined.
+
+    Returns:
+      An integer scalar containing the the last rank that joined.
+    """
     return MPI_LIB.horovod_join()
 
 
diff --git a/horovod/torch/mpi_ops.py b/horovod/torch/mpi_ops.py
--- a/horovod/torch/mpi_ops.py
+++ b/horovod/torch/mpi_ops.py
@@ -942,7 +942,7 @@ def synchronize(handle):
         raise HorovodInternalError(e)
 
 
-def join(device=-1):
+def join(device=-1) -> int:
     """A function that indicates that the rank finished processing data.
 
     All ranks that did not call join() continue to process allreduce operations.
@@ -954,7 +954,12 @@ def join(device=-1):
     Returns:
         Id of the rank that joined last.
     """
+    output = torch.tensor(-1, dtype=torch.int, device=torch.device("cpu"))
     try:
-        return mpi_lib.horovod_torch_join(device)
+        handle = mpi_lib.horovod_torch_join(output, device)
     except RuntimeError as e:
         raise HorovodInternalError(e)
+
+    _handle_map[handle] = (None, output)
+
+    return synchronize(handle).item()


Test patch:
diff --git a/test/parallel/test_tensorflow.py b/test/parallel/test_tensorflow.py
--- a/test/parallel/test_tensorflow.py
+++ b/test/parallel/test_tensorflow.py
@@ -3864,9 +3864,9 @@ def test_horovod_join_allreduce(self):
         for dtype, dim, first_join_rank in itertools.product(dtypes, dims, first_join_ranks):
             with tf.device("/gpu:%d" % local_rank):
                 tensor = self.random_uniform(
-                        [17] * dim, -100, 100, dtype=dtype)
+                    [17] * dim, -100, 100, dtype=dtype)
                 if local_rank == first_join_rank:
-                    self.evaluate(hvd.join())
+                    ret = self.evaluate(hvd.join())
                 else:
                     summed = hvd.allreduce(tensor, average=False)
                     multiplied = tensor * (size-1)
@@ -3881,9 +3881,15 @@ def test_horovod_join_allreduce(self):
                     else:
                         return
                     diff = self.evaluate(max_difference)
-                    self.evaluate(hvd.join())
+                    ret = self.evaluate(hvd.join())
                     self.assertTrue(diff <= threshold,
-                             "hvd.join with hvd.allreduce on GPU produces incorrect results")
+                                    "hvd.join with hvd.allreduce on GPU produces incorrect results")
+
+                self.assertNotEqual(ret, first_join_rank,
+                                    msg="The return value of hvd.join() may not be equal to first_join_rank")
+                ret_values = hvd.allgather_object(ret)
+                self.assertSequenceEqual(ret_values, [ret] * size,
+                                         msg="hvd.join() did not return the same value on each rank")
 
     def test_horovod_syncbn_gpu(self):
         """Test that the SyncBatchNormalization implementation is correct on GPU."""
diff --git a/test/parallel/test_torch.py b/test/parallel/test_torch.py
--- a/test/parallel/test_torch.py
+++ b/test/parallel/test_torch.py
@@ -2785,6 +2785,12 @@ def div(t, s):
                 assert torch.allclose(averaged_b, div(tensor_b * (size - 1), size), threshold), \
                     'hvd.join with hvd.allreduce produces incorrect results'
 
+            self.assertNotEqual(ret, first_join_rank,
+                                msg="The return value of hvd.join() may not be equal to first_join_rank")
+            ret_values = hvd.allgather_object(ret)
+            self.assertSequenceEqual(ret_values, [ret] * size,
+                                     msg="hvd.join() did not return the same value on each rank")
+
     def test_horovod_join_allgather(self):
         """Test Join op with allgather."""
         hvd.init()
@@ -2812,6 +2818,12 @@ def test_horovod_join_allgather(self):
 
             ret = hvd.join(hvd.local_rank())
 
+        self.assertNotEqual(ret, 0,
+                            msg="The return value of hvd.join() may not be equal to 0 because that would be the first rank to join")
+        ret_values = hvd.allgather_object(ret)
+        self.assertSequenceEqual(ret_values, [ret] * size,
+                                 msg="hvd.join() did not return the same value on each rank")
+
     def test_horovod_join_broadcast(self):
         """Test Join op with broadcast."""
         hvd.init()
@@ -2838,7 +2850,13 @@ def test_horovod_join_broadcast(self):
                 ret = hvd.join(hvd.local_rank())
             else:
                 ret = hvd.join()
-    
+
+        self.assertNotEqual(ret, 0,
+                            msg="The return value of hvd.join() may not be equal to 0 because that would be the first rank to join")
+        ret_values = hvd.allgather_object(ret)
+        self.assertSequenceEqual(ret_values, [ret] * size,
+                                 msg="hvd.join() did not return the same value on each rank")
+
     def test_horovod_sync_batch_norm(self):
         """Tests Horovod version of SyncBatchNorm."""
         if not torch.cuda.is_available():