Repository: horovod/horovod

Issue:
reducescatter() and grouped_reducescatter() crash for scalar inputs
`hvd.reducescatter(3.14)` currently leads to a C++ assertion failure in debug builds or a segmentation fault in release builds. 

There should either be a clear error message or it should behave similarly to `hvd.reducescatter([3.14])`, which returns a one-element tensor on the root rank and an empty tensor on the other ranks.


Discussion:

Resolving patch:
diff --git a/horovod/mxnet/mpi_ops.py b/horovod/mxnet/mpi_ops.py
--- a/horovod/mxnet/mpi_ops.py
+++ b/horovod/mxnet/mpi_ops.py
@@ -571,6 +571,8 @@ def reducescatter(tensor, op=Average, name=None, priority=0,
     """
     assert(isinstance(tensor, mx.nd.NDArray))
     assert(op in [Average, Sum])
+    if tensor.shape == ():
+        raise ValueError("reducescatter does not support scalar inputs")
     # Size of output is unknown, create output array that
     # will be resized during Horovod operation
     output = mx.nd.empty(shape=(1,), ctx=tensor.context,
@@ -625,6 +627,8 @@ def grouped_reducescatter(tensors, op=Average, name=None, priority=0,
     """
     assert(all(isinstance(t, mx.nd.NDArray) for t in tensors))
     assert(op in [Average, Sum])
+    if any(tensor.shape == () for tensor in tensors):
+        raise ValueError("groued_reducescatter does not support scalar inputs")
     # Sizes of outputs are unknown, create output arrays that
     # will be resized during Horovod operation
     outputs = [mx.nd.empty(shape=(1,), ctx=t.context, dtype=t.dtype)
diff --git a/horovod/torch/__init__.py b/horovod/torch/__init__.py
--- a/horovod/torch/__init__.py
+++ b/horovod/torch/__init__.py
@@ -51,6 +51,7 @@
     from horovod.torch.mpi_ops import nccl_built, ddl_built, ccl_built, cuda_built, rocm_built
     from horovod.torch.mpi_ops import ProcessSet, global_process_set, add_process_set, remove_process_set
     from horovod.torch.mpi_ops import Average, Sum, Adasum, Min, Max, Product
+    from horovod.torch.mpi_ops import HorovodInternalError
     from horovod.torch.optimizer import DistributedOptimizer
     from horovod.torch.sync_batch_norm import SyncBatchNorm
 


Test patch:
diff --git a/test/parallel/base_test_mxnet.py b/test/parallel/base_test_mxnet.py
--- a/test/parallel/base_test_mxnet.py
+++ b/test/parallel/base_test_mxnet.py
@@ -1791,6 +1791,19 @@ def test_horovod_reducescatter_average(self):
             assert almost_equal(averaged.asnumpy(), expected.asnumpy(), atol=threshold), \
                 f'hvd.reducescatter produces incorrect results: {hvd.rank()} {count} {dtype} {dim}'
 
+    def test_horovod_reducescatter_scalar_error(self):
+        if hvd.ccl_built():
+            self.skipTest("Reducescatter is not supported yet with oneCCL operations.")
+        if _is_mac and hvd.gloo_built() and not hvd.mpi_built():
+            self.skipTest("ReducescatterGloo is not supported on macOS")
+        hvd.init()
+        size = hvd.size()
+        rank = hvd.rank()
+        ctx = self._current_context()
+        scalar = mx.nd.array(rank, dtype=np.float32, ctx=ctx)
+        with self.assertRaises(ValueError):
+            _ = hvd.reducescatter(scalar, op=hvd.Average)
+
     def test_horovod_reducescatter_error(self):
         """Test that the reducescatter raises an error if different ranks try to
         send tensors of different rank or dimension."""
@@ -1970,6 +1983,19 @@ def test_horovod_grouped_reducescatter_average(self):
             assert all(almost_equal(t1.asnumpy(), t2.asnumpy(), atol=threshold) for t1, t2 in zip(averaged, expected)), \
                 f'hvd.grouped_reducescatter produces incorrect results: {hvd.rank()} {count} {dtype} {dim}'
 
+    def test_horovod_grouped_reducescatter_scalar_error(self):
+        if hvd.ccl_built():
+            self.skipTest("Reducescatter is not supported yet with oneCCL operations.")
+        if _is_mac and hvd.gloo_built() and not hvd.mpi_built():
+            self.skipTest("ReducescatterGloo is not supported on macOS")
+        hvd.init()
+        rank = hvd.rank()
+        ctx = self._current_context()
+        tensor_and_scalar = [mx.nd.zeros((3,1), ctx=ctx, dtype=np.float32),
+                             mx.nd.array(rank, dtype=np.float32, ctx=ctx)]
+        with self.assertRaises(ValueError):
+            _ = hvd.grouped_reducescatter(tensor_and_scalar, op=hvd.Average)
+
     def test_horovod_grouped_reducescatter_process_sets(self):
         """Test that grouped reducescatter correctly sums and scatters 1D, 2D, 3D tensors if restricted
         to non-global process sets."""
diff --git a/test/parallel/test_tensorflow.py b/test/parallel/test_tensorflow.py
--- a/test/parallel/test_tensorflow.py
+++ b/test/parallel/test_tensorflow.py
@@ -3473,6 +3473,18 @@ def test_horovod_reducescatter_cpu_uneven_fused(self):
         self.assertTrue(successful,
                         "hvd.reducescatter produces incorrect results")
 
+    def test_horovod_reducescatter_scalar_error(self):
+        if hvd.ccl_built():
+            self.skipTest("Reducescatter is not supported yet with oneCCL operations.")
+        if _is_mac and hvd.gloo_built() and not hvd.mpi_built():
+            self.skipTest("ReducescatterGloo is not supported on macOS")
+        hvd.init()
+        rank = hvd.rank()
+        with tf.device("/cpu:0"):
+            scalar = tf.constant(rank, dtype=tf.float32)
+            with self.assertRaises((tf.errors.InvalidArgumentError, ValueError, tf.errors.FailedPreconditionError)):
+                self.evaluate(hvd.reducescatter(scalar))
+
     def test_horovod_reducescatter_gpu(self):
         """Test that the reducescatter works on GPUs."""
         if not tf.test.is_gpu_available(cuda_only=True):
@@ -3851,6 +3863,18 @@ def test_horovod_grouped_reducescatter_cpu(self):
             diff = self.evaluate(max_difference)
             self.assertTrue(diff <= threshold, "hvd.grouped_reducescatter produces incorrect results")
 
+    def test_horovod_grouped_reducescatter_scalar_error(self):
+        if hvd.ccl_built():
+            self.skipTest("Reducescatter is not supported yet with oneCCL operations.")
+        if _is_mac and hvd.gloo_built() and not hvd.mpi_built():
+            self.skipTest("ReducescatterGloo is not supported on macOS")
+        hvd.init()
+        rank = hvd.rank()
+        with tf.device("/cpu:0"):
+            tensor_and_scalar = [tf.zeros((2,1), dtype=tf.float32), tf.constant(rank, dtype=tf.float32)]
+            with self.assertRaises((tf.errors.InvalidArgumentError, ValueError, tf.errors.FailedPreconditionError)):
+                self.evaluate(hvd.grouped_reducescatter(tensor_and_scalar))
+
     def test_horovod_grouped_reducescatter_grad_cpu(self):
         """Test the correctness of the grouped reducescatter gradient on CPU."""
         if hvd.ccl_built():
@@ -3987,7 +4011,6 @@ def test_horovod_grouped_allgather_grad_cpu(self):
         dtypes = [tf.float32, tf.float64]
         dims = [1, 2, 3]
         for dtype, dim in itertools.product(dtypes, dims):
-            print(dtype)
             tensor_sizes = [3, 2, 7, 4, 6, 8, 10] * 5
             tensor_sizes = tensor_sizes[:size]
 
diff --git a/test/parallel/test_torch.py b/test/parallel/test_torch.py
--- a/test/parallel/test_torch.py
+++ b/test/parallel/test_torch.py
@@ -3610,6 +3610,16 @@ def test_horovod_reducescatter_average(self):
             max_difference = averaged.data.sub(expected).max()
             assert max_difference <= threshold, 'hvd.reducescatter produces incorrect results'
 
+    def test_horovod_reducescatter_scalar_error(self):
+        if hvd.ccl_built():
+            self.skipTest("Reducescatter is not supported yet with oneCCL operations.")
+        if _is_mac and hvd.gloo_built() and not hvd.mpi_built():
+            self.skipTest("ReducescatterGloo is not supported on macOS")
+        hvd.init()
+        rank = hvd.rank()
+        scalar = self.cast_and_place(torch.tensor(rank), torch.FloatTensor)
+        with self.assertRaises((torch.FatalError, RuntimeError, hvd.HorovodInternalError)):
+            _ = hvd.reducescatter(scalar, op=hvd.Average)
 
     def test_horovod_reducescatter_adasum(self):
         """Test that the reducescatter raises an error if we use Adasum operation."""
@@ -3986,8 +3996,6 @@ def test_horovod_grouped_reducescatter(self):
                        torch.cuda.HalfTensor]
         dims = [1, 2, 3]
         for dtype, dim in itertools.product(dtypes, dims):
-            if rank == 0:
-                print(dtype, dim)
             torch.manual_seed(1234)
             tensors = [torch.FloatTensor(*([size * 4] * dim)).random_(-100, 100) for _ in range(5)]
             tensors = [self.cast_and_place(t, dtype) for t in tensors]
@@ -4052,6 +4060,18 @@ def test_horovod_grouped_reducescatter_average(self):
             assert all([torch.allclose(t1, t2, threshold) for t1, t2 in zip(expected, averaged)]), \
                 'hvd.grouped_reducescatter produces incorrect results for average'
 
+    def test_horovod_grouped_reducescatter_scalar_error(self):
+        if hvd.ccl_built():
+            self.skipTest("Reducescatter is not supported yet with oneCCL operations.")
+        if _is_mac and hvd.gloo_built() and not hvd.mpi_built():
+            self.skipTest("ReducescatterGloo is not supported on macOS")
+        hvd.init()
+        rank = hvd.rank()
+        scalar = self.cast_and_place(torch.tensor(rank), torch.FloatTensor)
+        tensor = self.cast_and_place(torch.zeros((3,1)), torch.FloatTensor)
+        with self.assertRaises((torch.FatalError, RuntimeError, hvd.HorovodInternalError)):
+            _ = hvd.grouped_reducescatter([tensor, scalar])
+
     def test_horovod_grouped_reducescatter_process_sets(self):
         """Test that grouped reducescatter correctly sums and scatters 1D, 2D, 3D tensors if restricted to process sets."""
         if hvd.ccl_built():