jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def test_horovod_alltoall_zero_splits_cpu(self):
        """Test alltoall with some ranks not participating / splits set to zero."""
        hvd.init()

        if hvd.size() == 1:
            self.skipTest("Only one worker available")

        active_ranks = range(0, hvd.size() // 2)
        silent_ranks = range(hvd.size() // 2, hvd.size())

        active_splits = [1 if r in active_ranks else 0 for r in range(hvd.size())]
        active_shape = [sum(active_splits), 4]
        silent_splits = [0] * hvd.size()
        silent_shape = [0, 4]

        with tf.device("/cpu:0"):
            if hvd.rank() in active_ranks:
                source_tensor = tf.fill(active_shape, value=tf.to_int32(hvd.rank()))
                splits = tf.convert_to_tensor(active_splits)
            else:
                source_tensor = tf.fill(silent_shape, value=tf.to_int32(hvd.rank()))
                splits = tf.convert_to_tensor(silent_splits)
            collected, received_splits = hvd.alltoall(source_tensor, splits, name="alltoall_zero_splits")
            result = self.evaluate(collected)

        if hvd.rank() in active_ranks:
            expected_result_shape = active_shape
        else:
            expected_result_shape = silent_shape
        self.assertSequenceEqual(result.shape, expected_result_shape)
        if hvd.rank() in active_ranks:
            for r_idx, r in enumerate(active_ranks):
                self.assertTrue(np.all(result[r_idx, ...] == r))
        else:
            self.assertLen(result, 0)
        if hvd.rank() in active_ranks:
            expected_rsplits = active_splits
        else:
            expected_rsplits = silent_splits
        self.assertSequenceEqual(self.evaluate(received_splits).tolist(), expected_rsplits,
                                 "hvd.alltoall returned incorrect received_splits")
//  [python]
def test_horovod_alltoall_zero_splits_cpu(self):
        """Test alltoall with some ranks not participating / splits set to zero."""
        hvd.init()

        if hvd.size() == 1:
            self.skipTest("Only one worker available")

        active_ranks = range(0, hvd.size() // 2)
        silent_ranks = range(hvd.size() // 2, hvd.size())

        active_splits = [1 if r in active_ranks else 0 for r in range(hvd.size())]
        active_shape = [sum(active_splits), 4]
        silent_splits = [0] * hvd.size()
        silent_shape = [0, 4]

        with tf.device("/cpu:0"):
            if hvd.rank() in active_ranks:
                source_tensor = tf.fill(active_shape, value=tf.cast(hvd.rank(), tf.int32))
                splits = tf.convert_to_tensor(active_splits)
            else:
                source_tensor = tf.fill(silent_shape, value=tf.cast(hvd.rank(), tf.int32))
                splits = tf.convert_to_tensor(silent_splits)
            collected, received_splits = hvd.alltoall(source_tensor, splits, name="alltoall_zero_splits")
            result = self.evaluate(collected)

        if hvd.rank() in active_ranks:
            expected_result_shape = active_shape
        else:
            expected_result_shape = silent_shape
        self.assertSequenceEqual(result.shape, expected_result_shape)
        if hvd.rank() in active_ranks:
            for r_idx, r in enumerate(active_ranks):
                self.assertTrue(np.all(result[r_idx, ...] == r))
        else:
            self.assertLen(result, 0)
        if hvd.rank() in active_ranks:
            expected_rsplits = active_splits
        else:
            expected_rsplits = silent_splits
        self.assertSequenceEqual(self.evaluate(received_splits).tolist(), expected_rsplits,
                                 "hvd.alltoall returned incorrect received_splits")
//  [python]
def test_horovod_reducescatter_cpu(self):
        """Test on CPU that the reducescatter correctly sums or averages and scatters 1D, 2D, 3D tensors."""
        if hvd.ccl_built():
            self.skipTest("Reducescatter is not supported yet with oneCCL operations.")
        if _is_mac and hvd.gloo_built() and not hvd.mpi_built():
            self.skipTest("ReducescatterGloo is not supported on macOS")
        hvd.init()
        rank = hvd.rank()
        size = hvd.size()
        dtypes = self.filter_supported_types([tf.int32, tf.int64, tf.float16, tf.float32, tf.float64])
        dims = [1, 2, 3]
        for red_op, dtype, dim in itertools.product([hvd.Sum, hvd.Average], dtypes, dims):
            with tf.device("/cpu:0"):
                tensor = self.random_uniform(
                    [size * 4] * dim, -100, 100, dtype=dtype)
                reduced = hvd.reducescatter(tensor, op=red_op)
            if red_op == hvd.Sum:
                expected = tf.cast(tensor[rank * 4:(rank + 1) * 4] * size, reduced.dtype)
            elif red_op == hvd.Average:
                expected = tf.cast(tensor[rank * 4:(rank + 1) * 4], reduced.dtype)
            max_difference = tf.reduce_max(tf.abs(reduced - expected))

            # Threshold for floating point equality depends on number of
            # ranks, since we're comparing against precise multiplication.
            if dtype == tf.float16:
                threshold = .5
            elif dtype in [tf.int32, tf.int64]:
                threshold = 0
            elif size < 10:
                threshold = 1e-4
            elif size < 15:
                threshold = 5e-4
            else:
                break

            diff = self.evaluate(max_difference)
            self.assertTrue(diff <= threshold,
                            "hvd.reducescatter produces incorrect results")
//  [python]
def test_horovod_reducescatter_cpu_postscale(self):
        """Test on CPU that the reducescatter correctly sums and scatters 1D, 2D, 3D tensors with postscaling."""
        if hvd.ccl_built():
            self.skipTest("Reducescatter is not supported yet with oneCCL operations.")
        if _is_mac and hvd.gloo_built() and not hvd.mpi_built():
            self.skipTest("ReducescatterGloo is not supported on macOS")
        hvd.init()
        rank = hvd.rank()
        size = hvd.size()
        dtypes = self.filter_supported_types([tf.int32, tf.int64, tf.float16, tf.float32])
        int_types = [tf.uint8, tf.int8, tf.int32, tf.int64]
        dims = [1, 2, 3]
        np.random.seed(12345)
        for dtype, dim in itertools.product(dtypes, dims):
            with tf.device("/cpu:0"):
                factor = np.random.uniform()
                tensor = self.random_uniform([size * 4] * dim, -100, 100, dtype=dtype)
                reduced = hvd.reducescatter(tensor, op=hvd.Sum, postscale_factor=factor)

                multiplied = tensor * size
                # Scaling done in FP64 math for integer types, FP32 math for FP16 on CPU
                multiplied = t.cast(multiplied, tf.float32 if dtype == tf.float16 else
                                     tf.float64 if dtype in int_types else dtype)
                factor = t.cast(factor, tf.float32 if dtype == tf.float16 else
                                              tf.float64 if dtype in int_types else dtype)

                expected = t.cast(factor * multiplied[rank * 4:(rank + 1) * 4], reduced.dtype)
                max_difference = tf.reduce_max(tf.abs(reduced - expected))

            # Threshold for floating point equality depends on number of
            # ranks, since we're comparing against precise multiplication.
            if size <= 3 or dtype in int_types:
                threshold = 0
            elif size < 10:
                threshold = 1e-4
            elif size < 15:
                threshold = 5e-4
            else:
                break

            diff = self.evaluate(max_difference)
            self.assertTrue(diff <= threshold,
                            "hvd.reducescatter produces incorrect results")
//  [python]
def test_horovod_reducescatter_cpu_postscale(self):
        """Test on CPU that the reducescatter correctly sums and scatters 1D, 2D, 3D tensors with postscaling."""
        if hvd.ccl_built():
            self.skipTest("Reducescatter is not supported yet with oneCCL operations.")
        if _is_mac and hvd.gloo_built() and not hvd.mpi_built():
            self.skipTest("ReducescatterGloo is not supported on macOS")
        hvd.init()
        rank = hvd.rank()
        size = hvd.size()
        dtypes = self.filter_supported_types([tf.int32, tf.int64, tf.float16, tf.float32])
        int_types = [tf.uint8, tf.int8, tf.int32, tf.int64]
        dims = [1, 2, 3]
        np.random.seed(12345)
        for dtype, dim in itertools.product(dtypes, dims):
            with tf.device("/cpu:0"):
                factor = np.random.uniform()
                tensor = self.random_uniform([size * 4] * dim, -100, 100, dtype=dtype)
                reduced = hvd.reducescatter(tensor, op=hvd.Sum, postscale_factor=factor)

                multiplied = tensor * size
                # Scaling done in FP64 math for integer types, FP32 math for FP16 on CPU
                multiplied = tf.cast(multiplied, tf.float32 if dtype == tf.float16 else
                                     tf.float64 if dtype in int_types else dtype)
                factor = tf.convert_to_tensor(factor, tf.float32 if dtype == tf.float16 else
                                              tf.float64 if dtype in int_types else dtype)

                expected = tf.cast(factor * multiplied[rank * 4:(rank + 1) * 4], reduced.dtype)
                max_difference = tf.reduce_max(tf.abs(reduced - expected))

            # Threshold for floating point equality depends on number of
            # ranks, since we're comparing against precise multiplication.
            if size <= 3 or dtype in int_types:
                threshold = 0
            elif size < 10:
                threshold = 1e-4
            elif size < 15:
                threshold = 5e-4
            else:
                break

            diff = self.evaluate(max_difference)
            self.assertTrue(diff <= threshold,
                            "hvd.reducescatter produces incorrect results")
//  [python]
def test_horovod_alltoall_zero_splits_gpu(self):
        """Test alltoall with some ranks not participating / splits set to zero."""
        # ncclCommInitRank failed: invalid usage
        hvd.init()

        if hvd.size() == 1:
            self.skipTest("Only one worker available")

        # Only do this test if there are GPUs available.
        if not tf.test.is_gpu_available(cuda_only=True):
            self.skipTest("No GPUs available")

        if int(os.environ.get('HOROVOD_MIXED_INSTALL', 0)):
            # Skip if compiled with CUDA but without HOROVOD_GPU_OPERATIONS.
            self.skipTest("Not compiled with HOROVOD_GPU_OPERATIONS")

        # This test does not apply if NCCL version < 2.7.0
        if hvd.nccl_built() and hvd.nccl_built() < 2700:
            self.skipTest("NCCL-based Alltoall requires NCCL version >= 2.7.0.")

        active_ranks = range(0, hvd.size() // 2)
        silent_ranks = range(hvd.size() // 2, hvd.size())

        active_splits = [1 if r in active_ranks else 0 for r in range(hvd.size())]
        active_shape = [sum(active_splits), 4]
        silent_splits = [0] * hvd.size()
        silent_shape = [0, 4]

        with tf.device("/gpu:%s" % hvd.local_rank()):
            if hvd.rank() in active_ranks:
                source_tensor = tf.fill(active_shape, value=t.cast(hvd.rank(), t.int32))
                splits = tf.convert_to_tensor(active_splits)
            else:
                source_tensor = tf.fill(silent_shape, value=t.cast(hvd.rank(), t.int32))
                splits = tf.convert_to_tensor(silent_splits)
            collected, received_splits = hvd.alltoall(source_tensor, splits, name="alltoall_zero_splits")
            result = self.evaluate(collected)

        if hvd.rank() in active_ranks:
            expected_result_shape = active_shape
        else:
            expected_result_shape = silent_shape
        self.assertSequenceEqual(result.shape, expected_result_shape)
        if hvd.rank() in active_ranks:
            for r_idx, r in enumerate(active_ranks):
                self.assertTrue(np.all(result[r_idx, ...] == r))
        else:
            self.assertLen(result, 0)
        if hvd.rank() in active_ranks:
            expected_rsplits = active_splits
        else:
            expected_rsplits = silent_splits
        self.assertSequenceEqual(self.evaluate(received_splits).tolist(), expected_rsplits,
                                 "hvd.alltoall returned incorrect received_splits")
//  [python]
def test_horovod_alltoall_zero_splits_gpu(self):
        """Test alltoall with some ranks not participating / splits set to zero."""
        # ncclCommInitRank failed: invalid usage
        hvd.init()

        if hvd.size() == 1:
            self.skipTest("Only one worker available")

        # Only do this test if there are GPUs available.
        if not tf.test.is_gpu_available(cuda_only=True):
            self.skipTest("No GPUs available")

        if int(os.environ.get('HOROVOD_MIXED_INSTALL', 0)):
            # Skip if compiled with CUDA but without HOROVOD_GPU_OPERATIONS.
            self.skipTest("Not compiled with HOROVOD_GPU_OPERATIONS")

        # This test does not apply if NCCL version < 2.7.0
        if hvd.nccl_built() and hvd.nccl_built() < 2700:
            self.skipTest("NCCL-based Alltoall requires NCCL version >= 2.7.0.")

        active_ranks = range(0, hvd.size() // 2)
        silent_ranks = range(hvd.size() // 2, hvd.size())

        active_splits = [1 if r in active_ranks else 0 for r in range(hvd.size())]
        active_shape = [sum(active_splits), 4]
        silent_splits = [0] * hvd.size()
        silent_shape = [0, 4]

        with tf.device("/gpu:%s" % hvd.local_rank()):
            if hvd.rank() in active_ranks:
                source_tensor = tf.fill(active_shape, value=tf.cast(hvd.rank(), tf.int32))
                splits = tf.convert_to_tensor(active_splits)
            else:
                source_tensor = tf.fill(silent_shape, value=tf.cast(hvd.rank(), tf.int32))
                splits = tf.convert_to_tensor(silent_splits)
            collected, received_splits = hvd.alltoall(source_tensor, splits, name="alltoall_zero_splits")
            result = self.evaluate(collected)

        if hvd.rank() in active_ranks:
            expected_result_shape = active_shape
        else:
            expected_result_shape = silent_shape
        self.assertSequenceEqual(result.shape, expected_result_shape)
        if hvd.rank() in active_ranks:
            for r_idx, r in enumerate(active_ranks):
                self.assertTrue(np.all(result[r_idx, ...] == r))
        else:
            self.assertLen(result, 0)
        if hvd.rank() in active_ranks:
            expected_rsplits = active_splits
        else:
            expected_rsplits = silent_splits
        self.assertSequenceEqual(self.evaluate(received_splits).tolist(), expected_rsplits,
                                 "hvd.alltoall returned incorrect received_splits")
//  [python]
def test_horovod_reducescatter_cpu_prescale(self):
        """Test on CPU that the reducescatter correctly sums and scatters 1D, 2D, 3D tensors with prescaling."""
        if hvd.ccl_built():
            self.skipTest("Reducescatter is not supported yet with oneCCL operations.")
        if _is_mac and hvd.gloo_built() and not hvd.mpi_built():
            self.skipTest("ReducescatterGloo is not supported on macOS")
        hvd.init()
        rank = hvd.rank()
        size = hvd.size()
        dtypes = self.filter_supported_types([tf.int32, tf.int64, tf.float16, tf.float32])
        int_types = [tf.uint8, tf.int8, tf.int32, tf.int64]
        dims = [1, 2, 3]
        np.random.seed(12345)
        for dtype, dim in itertools.product(dtypes, dims):
            with tf.device("/cpu:0"):
                factor = np.random.uniform()
                tensor = self.random_uniform([size * 4] * dim, -100, 100, dtype=dtype)
                reduced = hvd.reducescatter(tensor, op=hvd.Sum, prescale_factor=factor)

                # Scaling done in FP64 math for integer types, FP32 math for FP16 on CPU
                tensor = tf.cast(tensor, tf.float32 if dtype == tf.float16 else
                                 tf.float64 if dtype in int_types else dtype)
                factor = tf.cast(factor, tf.float32 if dtype == tf.float16 else
                                              tf.float64 if dtype in int_types else dtype)

                expected = tf.cast(factor * tensor[rank * 4:(rank + 1) * 4], reduced.dtype) * size
                max_difference = tf.reduce_max(tf.abs(reduced - expected))

            # Threshold for floating point equality depends on number of
            # ranks, since we're comparing against precise multiplication.
            if size <= 3 or dtype in int_types:
                threshold = 0
            elif size < 10:
                threshold = 1e-4
            elif size < 15:
                threshold = 5e-4
            else:
                break

            diff = self.evaluate(max_difference)
            self.assertTrue(diff <= threshold,
                            "hvd.reducescatter produces incorrect results")
//  [python]
def test_horovod_reducescatter_cpu_prescale(self):
        """Test on CPU that the reducescatter correctly sums and scatters 1D, 2D, 3D tensors with prescaling."""
        if hvd.ccl_built():
            self.skipTest("Reducescatter is not supported yet with oneCCL operations.")
        if _is_mac and hvd.gloo_built() and not hvd.mpi_built():
            self.skipTest("ReducescatterGloo is not supported on macOS")
        hvd.init()
        rank = hvd.rank()
        size = hvd.size()
        dtypes = self.filter_supported_types([tf.int32, tf.int64, tf.float16, tf.float32])
        int_types = [tf.uint8, tf.int8, tf.int32, tf.int64]
        dims = [1, 2, 3]
        np.random.seed(12345)
        for dtype, dim in itertools.product(dtypes, dims):
            with tf.device("/cpu:0"):
                factor = np.random.uniform()
                tensor = self.random_uniform([size * 4] * dim, -100, 100, dtype=dtype)
                reduced = hvd.reducescatter(tensor, op=hvd.Sum, prescale_factor=factor)

                # Scaling done in FP64 math for integer types, FP32 math for FP16 on CPU
                tensor = tf.cast(tensor, tf.float32 if dtype == tf.float16 else
                                 tf.float64 if dtype in int_types else dtype)
                factor = tf.convert_to_tensor(factor, tf.float32 if dtype == tf.float16 else
                                              tf.float64 if dtype in int_types else dtype)

                expected = tf.cast(factor * tensor[rank * 4:(rank + 1) * 4], reduced.dtype) * size
                max_difference = tf.reduce_max(tf.abs(reduced - expected))

            # Threshold for floating point equality depends on number of
            # ranks, since we're comparing against precise multiplication.
            if size <= 3 or dtype in int_types:
                threshold = 0
            elif size < 10:
                threshold = 1e-4
            elif size < 15:
                threshold = 5e-4
            else:
                break

            diff = self.evaluate(max_difference)
            self.assertTrue(diff <= threshold,
                            "hvd.reducescatter produces incorrect results")
//  [python]
def test_horovod_reducescatter_gpu(self):
        """Test that the reducescatter works on GPUs."""
        if not tf.test.is_gpu_available(cuda_only=True):
            self.skipTest("No GPUs available")

        if int(os.environ.get('HOROVOD_MIXED_INSTALL', 0)):
            # Skip if compiled with CUDA but without HOROVOD_GPU_OPERATIONS.
            self.skipTest("Not compiled with HOROVOD_GPU_OPERATIONS")

        hvd.init()
        local_rank = hvd.local_rank()
        rank = hvd.rank()
        size = hvd.size()

        dtypes = [tf.int32, tf.int64, tf.float16, tf.float32, tf.float64]
        dims = [1, 2, 3]
        for red_op, dtype, dim in itertools.product([hvd.Sum, hvd.Average], dtypes, dims):
            with tf.device("/gpu:%d" % local_rank):
                tensor = self.random_uniform(
                    [size * 4] * dim, -100, 100, dtype=dtype)
                reduced = hvd.reducescatter(tensor, op=red_op)
            if red_op == hvd.Sum:
                expected = tf.cast(tensor[rank * 4:(rank + 1) * 4] * size, reduced.dtype)
            elif red_op == hvd.Average:
                expected = tf.cast(tensor[rank * 4:(rank + 1) * 4], reduced.dtype)
            max_difference = tf.reduce_max(tf.abs(reduced - expected))

            # Threshold for floating point equality depends on number of
            # ranks, since we're comparing against precise multiplication.
            if dtype == tf.float16:
                threshold = .5
            elif dtype in [tf.int32, tf.int64]:
                threshold = 0
            elif size < 10:
                threshold = 1e-4
            elif size < 15:
                threshold = 5e-4
            else:
                return

            diff = self.evaluate(max_difference)
            self.assertTrue(diff <= threshold,
                            "hvd.reducescatter on GPU produces incorrect results")
//  [python]
def test_horovod_reducescatter_gpu_prescale(self):
        """Test that the reducescatter works on GPUs with prescaling."""
        if not tf.test.is_gpu_available(cuda_only=True):
            self.skipTest("No GPUs available")

        if int(os.environ.get('HOROVOD_MIXED_INSTALL', 0)):
            # Skip if compiled with CUDA but without HOROVOD_GPU_OPERATIONS.
            self.skipTest("Not compiled with HOROVOD_GPU_OPERATIONS")

        hvd.init()
        local_rank = hvd.local_rank()
        rank = hvd.rank()
        size = hvd.size()

        dtypes = self.filter_supported_types([tf.int32, tf.int64, tf.float16, tf.float32])
        int_types = [tf.uint8, tf.int8, tf.int32, tf.int64]
        dims = [1, 2, 3]
        np.random.seed(123456)
        for dtype, dim in itertools.product(dtypes, dims):
            with tf.device("/gpu:%d" % local_rank):
                factor = np.random.uniform()
                tensor = self.random_uniform([size * 4] * dim, -100, 100, dtype=dtype)
                reduced = hvd.reducescatter(tensor, op=hvd.Sum, prescale_factor=factor)

                # Scaling done in FP64 math for integer types
                tensor = t.cast(tensor, tf.float64 if dtype in int_types else dtype)
                factor = t.cast(factor, tf.float64 if dtype in int_types else dtype)
                expected = t.cast(factor * tensor[rank * 4:(rank + 1) * 4], reduced.dtype) * size
                max_difference = tf.reduce_max(tf.abs(reduced - expected))

            # Threshold for floating point equality depends on number of
            # ranks, since we're comparing against precise multiplication.
            if size <= 3 or dtype in int_types:
                threshold = 0
            elif size < 10:
                threshold = 1e-4
            elif size < 15:
                threshold = 5e-4
            else:
                break

            diff = self.evaluate(max_difference)
            self.assertTrue(diff <= threshold,
                            "hvd.reducescatter on GPU produces incorrect results")
//  [python]
def test_horovod_reducescatter_gpu_prescale(self):
        """Test that the reducescatter works on GPUs with prescaling."""
        if not tf.test.is_gpu_available(cuda_only=True):
            self.skipTest("No GPUs available")

        if int(os.environ.get('HOROVOD_MIXED_INSTALL', 0)):
            # Skip if compiled with CUDA but without HOROVOD_GPU_OPERATIONS.
            self.skipTest("Not compiled with HOROVOD_GPU_OPERATIONS")

        hvd.init()
        local_rank = hvd.local_rank()
        rank = hvd.rank()
        size = hvd.size()

        dtypes = self.filter_supported_types([tf.int32, tf.int64, tf.float16, tf.float32])
        int_types = [tf.uint8, tf.int8, tf.int32, tf.int64]
        dims = [1, 2, 3]
        np.random.seed(123456)
        for dtype, dim in itertools.product(dtypes, dims):
            with tf.device("/gpu:%d" % local_rank):
                factor = np.random.uniform()
                tensor = self.random_uniform([size * 4] * dim, -100, 100, dtype=dtype)
                reduced = hvd.reducescatter(tensor, op=hvd.Sum, prescale_factor=factor)

                # Scaling done in FP64 math for integer types
                tensor = tf.cast(tensor, tf.float64 if dtype in int_types else dtype)
                factor = tf.convert_to_tensor(factor, tf.float64 if dtype in int_types else dtype)
                expected = tf.cast(factor * tensor[rank * 4:(rank + 1) * 4], reduced.dtype) * size
                max_difference = tf.reduce_max(tf.abs(reduced - expected))

            # Threshold for floating point equality depends on number of
            # ranks, since we're comparing against precise multiplication.
            if size <= 3 or dtype in int_types:
                threshold = 0
            elif size < 10:
                threshold = 1e-4
            elif size < 15:
                threshold = 5e-4
            else:
                break

            diff = self.evaluate(max_difference)
            self.assertTrue(diff <= threshold,
                            "hvd.reducescatter on GPU produces incorrect results")
//  [python]
def test_horovod_reducescatter_gpu_postscale(self):
        """Test on GPU that the reducescatter correctly sums and scatters 1D, 2D, 3D tensors with postscaling."""
        if not tf.test.is_gpu_available(cuda_only=True):
            self.skipTest("No GPUs available")

        if int(os.environ.get('HOROVOD_MIXED_INSTALL', 0)):
            # Skip if compiled with CUDA but without HOROVOD_GPU_OPERATIONS.
            self.skipTest("Not compiled with HOROVOD_GPU_OPERATIONS")
        hvd.init()
        rank = hvd.rank()
        local_rank = hvd.local_rank()
        size = hvd.size()
        dtypes = self.filter_supported_types([tf.int32, tf.int64, tf.float16, tf.float32])
        int_types = [tf.uint8, tf.int8, tf.int32, tf.int64]
        dims = [1, 2, 3]
        np.random.seed(12345)
        for dtype, dim in itertools.product(dtypes, dims):
            with tf.device("/gpu:%s" % local_rank):
                factor = np.random.uniform()
                tensor = self.random_uniform([size * 4] * dim, -100, 100, dtype=dtype)
                reduced = hvd.reducescatter(tensor, op=hvd.Sum, postscale_factor=factor)

                multiplied = tensor * size
                # Scaling done in FP64 math for integer types.
                multiplied = tf.cast(multiplied, tf.float64 if dtype in int_types else dtype)
                factor = tf.cast(factor, tf.float64 if dtype in int_types else dtype)

                expected = tf.cast(factor * multiplied[rank * 4:(rank + 1) * 4], reduced.dtype)
                max_difference = tf.reduce_max(tf.abs(reduced - expected))

            # Threshold for floating point equality depends on number of
            # ranks, since we're comparing against precise multiplication.
            if size <= 3 or dtype in int_types:
                threshold = 0
            elif size < 10:
                threshold = 1e-4
            elif size < 15:
                threshold = 5e-4
            else:
                break

            diff = self.evaluate(max_difference)
            self.assertTrue(diff <= threshold,
                            "hvd.reducescatter produces incorrect results")
//  [python]
def test_horovod_reducescatter_gpu_postscale(self):
        """Test on GPU that the reducescatter correctly sums and scatters 1D, 2D, 3D tensors with postscaling."""
        if not tf.test.is_gpu_available(cuda_only=True):
            self.skipTest("No GPUs available")

        if int(os.environ.get('HOROVOD_MIXED_INSTALL', 0)):
            # Skip if compiled with CUDA but without HOROVOD_GPU_OPERATIONS.
            self.skipTest("Not compiled with HOROVOD_GPU_OPERATIONS")
        hvd.init()
        rank = hvd.rank()
        local_rank = hvd.local_rank()
        size = hvd.size()
        dtypes = self.filter_supported_types([tf.int32, tf.int64, tf.float16, tf.float32])
        int_types = [tf.uint8, tf.int8, tf.int32, tf.int64]
        dims = [1, 2, 3]
        np.random.seed(12345)
        for dtype, dim in itertools.product(dtypes, dims):
            with tf.device("/gpu:%s" % local_rank):
                factor = np.random.uniform()
                tensor = self.random_uniform([size * 4] * dim, -100, 100, dtype=dtype)
                reduced = hvd.reducescatter(tensor, op=hvd.Sum, postscale_factor=factor)

                multiplied = tensor * size
                # Scaling done in FP64 math for integer types.
                multiplied = tf.cast(multiplied, tf.float64 if dtype in int_types else dtype)
                factor = tf.convert_to_tensor(factor, tf.float64 if dtype in int_types else dtype)

                expected = tf.cast(factor * multiplied[rank * 4:(rank + 1) * 4], reduced.dtype)
                max_difference = tf.reduce_max(tf.abs(reduced - expected))

            # Threshold for floating point equality depends on number of
            # ranks, since we're comparing against precise multiplication.
            if size <= 3 or dtype in int_types:
                threshold = 0
            elif size < 10:
                threshold = 1e-4
            elif size < 15:
                threshold = 5e-4
            else:
                break

            diff = self.evaluate(max_difference)
            self.assertTrue(diff <= threshold,
                            "hvd.reducescatter produces incorrect results")
//  [python]
def test_horovod_grouped_reducescatter_cpu(self):
        """Test on CPU that the grouped reducescatter correctly sums or averages and scatters 1D, 2D, 3D tensors."""
        if hvd.ccl_built():
            self.skipTest("Reducescatter is not supported yet with oneCCL operations.")
        if _is_mac and hvd.gloo_built() and not hvd.mpi_built():
            self.skipTest("ReducescatterGloo is not supported on macOS")
        hvd.init()
        rank = hvd.rank()
        size = hvd.size()
        dtypes = self.filter_supported_types([tf.int32, tf.int64, tf.float16, tf.float32, tf.float64])
        dims = [1, 2, 3]
        for red_op, dtype, dim in itertools.product([hvd.Sum, hvd.Average], dtypes, dims):
            with tf.device("/cpu:0"):
                tensors = [self.random_uniform([size * 4] * dim, -100, 100, dtype=dtype) for _ in range(5)]
                reduced = hvd.grouped_reducescatter(tensors, op=red_op)
            if red_op == hvd.Sum:
                expected = [tf.cast(tensor[rank * 4:(rank + 1) * 4] * size, reduced[0].dtype) for tensor in tensors]
            elif red_op == hvd.Average:
                expected = [tf.cast(tensor[rank * 4:(rank + 1) * 4], reduced[0].dtype) for tensor in tensors]
            max_difference = tf.reduce_max([tf.reduce_max(tf.abs(t1 - t2)) for t1, t2 in zip(reduced, expected)])

            # Threshold for floating point equality depends on number of
            # ranks, since we're comparing against precise multiplication.
            if dtype == tf.float16:
                threshold = .5
            elif dtype in [tf.int32, tf.int64]:
                threshold = 0
            elif size < 10:
                threshold = 1e-4
            elif size < 15:
                threshold = 5e-4
            else:
                break

            diff = self.evaluate(max_difference)
            self.assertTrue(diff <= threshold, "hvd.grouped_reducescatter produces incorrect results")
//  [python]
def test_horovod_grouped_reducescatter_cpu_prescale(self):
        """Test on CPU that the grouped reducescatter correctly sums and scatters 1D, 2D, 3D tensors with prescaling."""
        if hvd.ccl_built():
            self.skipTest("Reducescatter is not supported yet with oneCCL operations.")
        if _is_mac and hvd.gloo_built() and not hvd.mpi_built():
            self.skipTest("ReducescatterGloo is not supported on macOS")
        hvd.init()
        rank = hvd.rank()
        size = hvd.size()
        dtypes = self.filter_supported_types([tf.int32, tf.int64, tf.float16, tf.float32])
        int_types = [tf.uint8, tf.int8, tf.int32, tf.int64]
        dims = [1, 2, 3]
        np.random.seed(12345)
        for dtype, dim in itertools.product(dtypes, dims):
            with tf.device("/cpu:0"):
                factor = np.random.uniform()
                tensors = [self.random_uniform([size * 4] * dim, -100, 100, dtype=dtype) for _ in range(5)]
                reduced = hvd.grouped_reducescatter(tensors, op=hvd.Sum, prescale_factor=factor)

                # Scaling done in FP64 math for integer types, FP32 math for FP16 on CPU
                tensors = [tf.cast(t, tf.float32 if dtype == tf.float16 else
                           tf.float64 if dtype in int_types else dtype) for t in tensors]
                factor = tf.convert_to_tensor(factor, tf.float32 if dtype == tf.float16 else
                                              tf.float64 if dtype in int_types else dtype)

                expected = [tf.cast(factor * t[rank * 4:(rank + 1) * 4], reduced[0].dtype) * size
                            for t in tensors]
                max_difference = tf.reduce_max([tf.reduce_max(tf.abs(t1 - t2)) for t1, t2 in zip(reduced, expected)])

            # Threshold for floating point equality depends on number of
            # ranks, since we're comparing against precise multiplication.
            if size <= 3 or dtype in int_types:
                threshold = 0
            elif size < 10:
                threshold = 1e-4
            elif size < 15:
                threshold = 5e-4
            else:
                break

            diff = self.evaluate(max_difference)
            self.assertTrue(diff <= threshold, "hvd.grouped_reducescatter produces incorrect results")
//  [python]
def GELU():
    def gelu(x, approximate=False):
        if approximate:
            coeff = t.cast(0.044715, x.dtype)
            return 0.5 * x * (1.0 + tf.tanh(0.7978845608028654 * (x + coeff * tf.pow(x, 3))))
        else:
            return 0.5 * x * (1.0 + tf.math.erf(x / t.cast(1.4142135623730951, x.dtype)))

    return nn.Activation(gelu)
//  [python]
def GELU():
    def gelu(x, approximate=False):
        if approximate:
            coeff = tf.cast(0.044715, x.dtype)
            return 0.5 * x * (1.0 + tf.tanh(0.7978845608028654 * (x + coeff * tf.pow(x, 3))))
        else:
            return 0.5 * x * (1.0 + tf.math.erf(x / tf.cast(1.4142135623730951, x.dtype)))

    return nn.Activation(gelu)
//  [python]
def test_horovod_grouped_reducescatter_gpu(self):
        """Test that the grouped reducescatter works on GPUs."""
        if not tf.test.is_gpu_available(cuda_only=True):
            self.skipTest("No GPUs available")

        if int(os.environ.get('HOROVOD_MIXED_INSTALL', 0)):
            # Skip if compiled with CUDA but without HOROVOD_GPU_OPERATIONS.
            self.skipTest("Not compiled with HOROVOD_GPU_OPERATIONS")

        hvd.init()
        local_rank = hvd.local_rank()
        rank = hvd.rank()
        size = hvd.size()

        dtypes = [tf.int32, tf.int64, tf.float16, tf.float32, tf.float64]
        dims = [1, 2, 3]
        for red_op, dtype, dim in itertools.product([hvd.Sum, hvd.Average], dtypes, dims):
            with tf.device("/gpu:%d" % local_rank):
                tensors = [self.random_uniform([size * 4] * dim, -100, 100, dtype=dtype)
                           for _ in range(5)]
                reduced = hvd.grouped_reducescatter(tensors, op=red_op)
            if red_op == hvd.Sum:
                expected = [tf.cast(tensor[rank * 4:(rank + 1) * 4] * size, reduced[0].dtype)
                            for tensor in tensors]
            elif red_op == hvd.Average:
                expected = [tf.cast(tensor[rank * 4:(rank + 1) * 4], reduced[0].dtype)
                            for tensor in tensors]
            max_difference = tf.reduce_max([tf.reduce_max(tf.abs(t1 - t2)) for t1, t2 in zip(reduced, expected)])

            # Threshold for floating point equality depends on number of
            # ranks, since we're comparing against precise multiplication.
            if dtype == tf.float16:
                threshold = .5
            elif dtype in [tf.int32, tf.int64]:
                threshold = 0
            elif size < 10:
                threshold = 1e-4
            elif size < 15:
                threshold = 5e-4
            else:
                return

            diff = self.evaluate(max_difference)
            self.assertTrue(diff <= threshold,
                            "hvd.grouped_reducescatter on GPU produces incorrect results")
//  [python]
def test_horovod_grouped_reducescatter_gpu_prescale(self):
        """Test on GPU that the grouped reducescatter correctly sums and scatters 1D, 2D, 3D tensors with prescaling."""
        if not tf.test.is_gpu_available(cuda_only=True):
            self.skipTest("No GPUs available")
        if int(os.environ.get('HOROVOD_MIXED_INSTALL', 0)):
            # Skip if compiled with CUDA but without HOROVOD_GPU_OPERATIONS.
            self.skipTest("Not compiled with HOROVOD_GPU_OPERATIONS")
        hvd.init()
        local_rank = hvd.local_rank()
        rank = hvd.rank()
        size = hvd.size()
        dtypes = self.filter_supported_types([tf.int32, tf.int64, tf.float16, tf.float32])
        int_types = [tf.uint8, tf.int8, tf.int32, tf.int64]
        dims = [1, 2, 3]
        np.random.seed(12345)
        for dtype, dim in itertools.product(dtypes, dims):
            with tf.device("/gpu:%d" % local_rank):
                factor = np.random.uniform()
                tensors = [self.random_uniform([size * 4] * dim, -100, 100, dtype=dtype) for _ in range(5)]
                reduced = hvd.grouped_reducescatter(tensors, op=hvd.Sum, prescale_factor=factor)

                # Scaling done in FP64 math for integer types
                tensors = [tf.cast(t, tf.float64) if dtype in int_types else t for t in tensors]
                factor = tf.convert_to_tensor(factor, tf.float64 if dtype in int_types else dtype)

                expected = [tf.cast(factor * t[rank * 4:(rank + 1) * 4], reduced[0].dtype) * size
                            for t in tensors]
                max_difference = tf.reduce_max([tf.reduce_max(tf.abs(t1 - t2)) for t1, t2 in zip(reduced, expected)])

            # Threshold for floating point equality depends on number of
            # ranks, since we're comparing against precise multiplication.
            if size <= 3 or dtype in int_types:
                threshold = 0
            elif size < 10:
                threshold = 1e-4
            elif size < 15:
                threshold = 5e-4
            else:
                break

            diff = self.evaluate(max_difference)
            self.assertTrue(diff <= threshold, "hvd.grouped_reducescatter produces incorrect results")
//  [python]
def test_horovod_grouped_reducescatter_gpu_prescale(self):
        """Test on GPU that the grouped reducescatter correctly sums and scatters 1D, 2D, 3D tensors with prescaling."""
        if not tf.test.is_gpu_available(cuda_only=True):
            self.skipTest("No GPUs available")
        if int(os.environ.get('HOROVOD_MIXED_INSTALL', 0)):
            # Skip if compiled with CUDA but without HOROVOD_GPU_OPERATIONS.
            self.skipTest("Not compiled with HOROVOD_GPU_OPERATIONS")
        hvd.init()
        local_rank = hvd.local_rank()
        rank = hvd.rank()
        size = hvd.size()
        dtypes = self.filter_supported_types([tf.int32, tf.int64, tf.float16, tf.float32])
        int_types = [tf.uint8, tf.int8, tf.int32, tf.int64]
        dims = [1, 2, 3]
        np.random.seed(12345)
        for dtype, dim in itertools.product(dtypes, dims):
            with tf.device("/gpu:%d" % local_rank):
                factor = np.random.uniform()
                tensors = [self.random_uniform([size * 4] * dim, -100, 100, dtype=dtype) for _ in range(5)]
                reduced = hvd.grouped_reducescatter(tensors, op=hvd.Sum, prescale_factor=factor)

                # Scaling done in FP64 math for integer types
                tensors = [tf.cast(t, tf.float64 if dtype in int_types else dtype) for t in tensors]
                factor = tf.convert_to_tensor(factor, tf.float64 if dtype in int_types else dtype)

                expected = [tf.cast(factor * t[rank * 4:(rank + 1) * 4], reduced[0].dtype) * size
                            for t in tensors]
                max_difference = tf.reduce_max([tf.reduce_max(tf.abs(t1 - t2)) for t1, t2 in zip(reduced, expected)])

            # Threshold for floating point equality depends on number of
            # ranks, since we're comparing against precise multiplication.
            if size <= 3 or dtype in int_types:
                threshold = 0
            elif size < 10:
                threshold = 1e-4
            elif size < 15:
                threshold = 5e-4
            else:
                break

            diff = self.evaluate(max_difference)
            self.assertTrue(diff <= threshold, "hvd.grouped_reducescatter produces incorrect results")
//  [python]
def test_horovod_grouped_reducescatter_cpu_postscale(self):
        """Test on CPU that the grouped reducescatter correctly sums and scatters 1D, 2D, 3D tensors with postscaling."""
        if hvd.ccl_built():
            self.skipTest("Reducescatter is not supported yet with oneCCL operations.")
        if _is_mac and hvd.gloo_built() and not hvd.mpi_built():
            self.skipTest("ReducescatterGloo is not supported on macOS")
        hvd.init()
        rank = hvd.rank()
        size = hvd.size()
        dtypes = self.filter_supported_types([tf.int32, tf.int64, tf.float16, tf.float32])
        int_types = [tf.uint8, tf.int8, tf.int32, tf.int64]
        dims = [1, 2, 3]
        np.random.seed(12345)
        for dtype, dim in itertools.product(dtypes, dims):
            with tf.device("/cpu:0"):
                factor = np.random.uniform()
                tensors = [self.random_uniform([size * 4] * dim, -100, 100, dtype=dtype) for _ in range(5)]
                reduced = hvd.grouped_reducescatter(tensors, op=hvd.Sum, postscale_factor=factor)

                multiplied = [t * size for t in tensors]
                # Scaling done in FP64 math for integer types, FP32 math for FP16 on CPU
                multiplied = [tf.cast(t, tf.float32 if dtype == tf.float16 else
                              tf.float64 if dtype in int_types else dtype) for t in multiplied]
                factor = tf.cast(factor, tf.float32 if dtype == tf.float16 else
                                              tf.float64 if dtype in int_types else dtype)

                expected = [tf.cast(factor * m[rank * 4:(rank + 1) * 4], reduced[0].dtype)
                            for m in multiplied]
                max_difference = tf.reduce_max([tf.reduce_max(tf.abs(t1 - t2)) for t1, t2 in zip(reduced, expected)])

            # Threshold for floating point equality depends on number of
            # ranks, since we're comparing against precise multiplication.
            if size <= 3 or dtype in int_types:
                threshold = 0
            elif size < 10:
                threshold = 1e-4
            elif size < 15:
                threshold = 5e-4
            else:
                break

            diff = self.evaluate(max_difference)
            self.assertTrue(diff <= threshold, "hvd.grouped_reducescatter produces incorrect results")
//  [python]
def test_horovod_grouped_reducescatter_cpu_postscale(self):
        """Test on CPU that the grouped reducescatter correctly sums and scatters 1D, 2D, 3D tensors with postscaling."""
        if hvd.ccl_built():
            self.skipTest("Reducescatter is not supported yet with oneCCL operations.")
        if _is_mac and hvd.gloo_built() and not hvd.mpi_built():
            self.skipTest("ReducescatterGloo is not supported on macOS")
        hvd.init()
        rank = hvd.rank()
        size = hvd.size()
        dtypes = self.filter_supported_types([tf.int32, tf.int64, tf.float16, tf.float32])
        int_types = [tf.uint8, tf.int8, tf.int32, tf.int64]
        dims = [1, 2, 3]
        np.random.seed(12345)
        for dtype, dim in itertools.product(dtypes, dims):
            with tf.device("/cpu:0"):
                factor = np.random.uniform()
                tensors = [self.random_uniform([size * 4] * dim, -100, 100, dtype=dtype) for _ in range(5)]
                reduced = hvd.grouped_reducescatter(tensors, op=hvd.Sum, postscale_factor=factor)

                multiplied = [t * size for t in tensors]
                # Scaling done in FP64 math for integer types, FP32 math for FP16 on CPU
                multiplied = [tf.cast(t, tf.float32 if dtype == tf.float16 else
                              tf.float64 if dtype in int_types else dtype) for t in multiplied]
                factor = tf.convert_to_tensor(factor, tf.float32 if dtype == tf.float16 else
                                              tf.float64 if dtype in int_types else dtype)

                expected = [tf.cast(factor * m[rank * 4:(rank + 1) * 4], reduced[0].dtype)
                            for m in multiplied]
                max_difference = tf.reduce_max([tf.reduce_max(tf.abs(t1 - t2)) for t1, t2 in zip(reduced, expected)])

            # Threshold for floating point equality depends on number of
            # ranks, since we're comparing against precise multiplication.
            if size <= 3 or dtype in int_types:
                threshold = 0
            elif size < 10:
                threshold = 1e-4
            elif size < 15:
                threshold = 5e-4
            else:
                break

            diff = self.evaluate(max_difference)
            self.assertTrue(diff <= threshold, "hvd.grouped_reducescatter produces incorrect results")
//  [python]
def test_horovod_grouped_reducescatter_gpu_postscale(self):
        """Test on GPU that the grouped reducescatter correctly sums and scatters 1D, 2D, 3D tensors with postscaling"""
        if not tf.test.is_gpu_available(cuda_only=True):
            self.skipTest("No GPUs available")
        if int(os.environ.get('HOROVOD_MIXED_INSTALL', 0)):
            # Skip if compiled with CUDA but without HOROVOD_GPU_OPERATIONS.
            self.skipTest("Not compiled with HOROVOD_GPU_OPERATIONS")
        hvd.init()
        local_rank = hvd.local_rank()
        rank = hvd.rank()
        size = hvd.size()
        dtypes = self.filter_supported_types([tf.int32, tf.int64, tf.float16, tf.float32])
        int_types = [tf.uint8, tf.int8, tf.int32, tf.int64]
        dims = [1, 2, 3]
        np.random.seed(12345)
        for dtype, dim in itertools.product(dtypes, dims):
            with tf.device("/gpu:%d" % local_rank):
                factor = np.random.uniform()
                tensors = [self.random_uniform([size * 4] * dim, -100, 100, dtype=dtype) for _ in range(5)]
                reduced = hvd.grouped_reducescatter(tensors, op=hvd.Sum, postscale_factor=factor)

                multiplied = [t * size for t in tensors]
                # Scaling done in FP64 math for integer types
                multiplied = [tf.cast(t, tf.float64 if dtype in int_types else dtype) for t in multiplied]
                factor = tf.convert_to_tensor(factor, tf.float64 if dtype in int_types else dtype)

                expected = [tf.cast(factor * m[rank * 4:(rank + 1) * 4], reduced[0].dtype)
                            for m in multiplied]
                max_difference = tf.reduce_max([tf.reduce_max(tf.abs(t1 - t2)) for t1, t2 in zip(reduced, expected)])

            # Threshold for floating point equality depends on number of
            # ranks, since we're comparing against precise multiplication.
            if size <= 3 or dtype in int_types:
                threshold = 0
            elif size < 10:
                threshold = 1e-4
            elif size < 15:
                threshold = 5e-4
            else:
                break

            diff = self.evaluate(max_difference)
            self.assertTrue(diff <= threshold, "hvd.grouped_reducescatter produces incorrect results")