# Buggy code
def allreduce(
    tensor,
    average=None,
    device_dense="",
    device_sparse="",
    compression=Compression.none,
    op=None,
):
    """Perform an allreduce on a tf.Tensor or tf.IndexedSlices.

    This function performs a bandwidth-optimal ring allreduce on the input
    tensor. If the input is an tf.IndexedSlices, the function instead does an
    allgather on the values and the indices, effectively doing an allreduce on
    the represented tensor.

    Arguments:
        tensor: tf.Tensor, tf.Variable, or tf.IndexedSlices to reduce.
                The shape of the input must be identical across all ranks.
        average: DEPRECATED, please use op instead.
        device_dense: Device to be used for dense tensors. Uses GPU by default
                      if Horovod was built with HOROVOD_GPU_ALLREDUCE.
        device_sparse: Device to be used for sparse tensors. Uses GPU by default
                       if Horovod was built with HOROVOD_GPU_ALLGATHER.
        compression: Compression algorithm used to reduce the amount of data
                     sent and received by each worker node.  Defaults to not
                     using compression.
        op: The reduction operation to combine tensors across different ranks.
            Defaults to Average if None is given.

    Returns:
        A tensor of the same shape and type as `tensor`, summed across all
        processes.
    """
    op = handle_average_backwards_compatibility(op, average)
    # Averaging happens in framework code, so translate that to Sum for the actual call
    true_op = Sum if op == Average else op

    if isinstance(tensor, tf.IndexedSlices):
        # TODO: Need to fix this to actuall call Adasum
        if op == Adasum:
            raise NotImplementedError(
                "The Adasum reduction does not currently support "
                "sparse tensors. As a workaround please pass sparse_as_dense=True to "
                "DistributedOptimizer"
            )
        with tf.device(device_sparse):
            # For IndexedSlices, do two allgathers instead of an allreduce.
            horovod_size = tf.cast(size(), tensor.values.dtype)
            values = allgather(tensor.values)
            indices = allgather(tensor.indices)

            # To make this operation into an average, divide allgathered values by
            # the Horovod size.
            new_values = (values / horovod_size) if op == Average else values
        return tf.IndexedSlices(new_values, indices, dense_shape=tensor.dense_shape)
    else:
        with tf.device(device_dense):
            horovod_size = tf.cast(size(), dtype=tensor.dtype)
            tensor_compressed, ctx = compression.compress(tensor)
            summed_tensor_compressed = _allreduce(tensor_compressed, op=true_op)
            summed_tensor = compression.decompress(summed_tensor_compressed, ctx)
            if op == Adasum:
                if "CPU" not in tensor.device and has_gpu:
                    if nccl_built():
                        if not is_homogeneous:
                            raise NotImplementedError(
                                "Running GPU Adasum on heterogeneous cluster is not supported yet."
                            )
                        elif not check_num_rank_power_of_2(int(size() / local_size())):
                            raise NotImplementedError(
                                "Running GPU Adasum with non-power of 2 nodes is not supported yet."
                            )
                        horovod_local_size = tf.cast(local_size(), dtype=tensor.dtype)
                        new_tensor = summed_tensor / horovod_local_size
                    else:
                        warnings.warn(
                            "Adasum reduction does not currently support "
                            "GPU reduction using MPI. Tensors are copied to CPU memory instead."
                            "To use Adasum for GPU reduction, please compile Horovod with HOROVOD_GPU_ALLREDUCE=NCCL."
                        )
                        new_tensor = summed_tensor
                else:
                    if not check_num_rank_power_of_2(size()):
                        raise NotImplementedError(
                            "Running Adasum with non-power of 2 ranks is not supported yet."
                        )
                    new_tensor = summed_tensor
            else:
                new_tensor = (
                    (summed_tensor / horovod_size) if op == Average else summed_tensor
                )
        return new_tensor


# Traceback (AssertionError)
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Checking whether extension tensorflow was built with MPI.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint8 = np.dtype([("qint8", np.int8, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_quint8 = np.dtype([("quint8", np.uint8, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint16 = np.dtype([("qint16", np.int16, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_quint16 = np.dtype([("quint16", np.uint16, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint32 = np.dtype([("qint32", np.int32, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   np_resource = np.dtype([("resource", np.ubyte, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint8 = np.dtype([("qint8", np.int8, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_quint8 = np.dtype([("quint8", np.uint8, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint16 = np.dtype([("qint16", np.int16, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_quint16 = np.dtype([("quint16", np.uint16, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint32 = np.dtype([("qint32", np.int32, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   np_resource = np.dtype([("resource", np.ubyte, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Traceback (most recent call last):
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 73, in _target_fn
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     ext = importlib.import_module('.' + ext_base_name, 'horovod')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/importlib/__init__.py", line 126, in import_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     return _bootstrap._gcd_import(name[level:], package, level)
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 994, in _gcd_import
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 971, in _find_and_load
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 955, in _find_and_load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 665, in _load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap_external>", line 678, in exec_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/tensorflow/__init__.py", line 43, in <module>
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     has_gpu = gpu_available('tensorflow')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 104, in gpu_available
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     ext_base_name, available_fn, 'running with GPU', verbose) or False
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 90, in _check_extension_lambda
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     p.start()
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/multiprocessing/process.py", line 103, in start
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     'daemonic processes are not allowed to have children'
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || AssertionError: daemonic processes are not allowed to have children
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Extension tensorflow was NOT built with MPI.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Checking whether extension torch was built with MPI.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Traceback (most recent call last):
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 73, in _target_fn
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     ext = importlib.import_module('.' + ext_base_name, 'horovod')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/importlib/__init__.py", line 126, in import_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     return _bootstrap._gcd_import(name[level:], package, level)
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 994, in _gcd_import
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 971, in _find_and_load
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 955, in _find_and_load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 665, in _load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap_external>", line 678, in exec_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/torch/__init__.py", line 39, in <module>
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     from horovod.torch.mpi_ops import allreduce, allreduce_async, allreduce_, allreduce_async_
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 80, in <module>
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     _has_gpu = gpu_available('torch')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 104, in gpu_available
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     ext_base_name, available_fn, 'running with GPU', verbose) or False
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 90, in _check_extension_lambda
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     p.start()
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/multiprocessing/process.py", line 103, in start
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     'daemonic processes are not allowed to have children'
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || AssertionError: daemonic processes are not allowed to have children
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Extension torch was NOT built with MPI.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Checking whether extension mxnet was built with MPI.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Traceback (most recent call last):
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 73, in _target_fn
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     ext = importlib.import_module('.' + ext_base_name, 'horovod')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/importlib/__init__.py", line 126, in import_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     return _bootstrap._gcd_import(name[level:], package, level)
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 994, in _gcd_import
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 971, in _find_and_load
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 955, in _find_and_load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 665, in _load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap_external>", line 678, in exec_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/mxnet/__init__.py", line 23, in <module>
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     __file__, 'mpi_lib')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 51, in check_extension
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     'Horovod with %s=1 to debug the build error.' % (ext_name, ext_env_var))
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Extension mxnet was NOT built with MPI.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || ImportError: Extension horovod.mxnet has not been built.  If this is not expected, reinstall Horovod with HOROVOD_WITH_MXNET=1 to debug the build error.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Checking whether extension tensorflow was built with Gloo.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint8 = np.dtype([("qint8", np.int8, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_quint8 = np.dtype([("quint8", np.uint8, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint16 = np.dtype([("qint16", np.int16, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_quint16 = np.dtype([("quint16", np.uint16, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint32 = np.dtype([("qint32", np.int32, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   np_resource = np.dtype([("resource", np.ubyte, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint8 = np.dtype([("qint8", np.int8, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_quint8 = np.dtype([("quint8", np.uint8, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint16 = np.dtype([("qint16", np.int16, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_quint16 = np.dtype([("quint16", np.uint16, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint32 = np.dtype([("qint32", np.int32, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   np_resource = np.dtype([("resource", np.ubyte, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Traceback (most recent call last):
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 73, in _target_fn
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     ext = importlib.import_module('.' + ext_base_name, 'horovod')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/importlib/__init__.py", line 126, in import_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     return _bootstrap._gcd_import(name[level:], package, level)
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 994, in _gcd_import
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 971, in _find_and_load
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 955, in _find_and_load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 665, in _load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap_external>", line 678, in exec_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/tensorflow/__init__.py", line 43, in <module>
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     has_gpu = gpu_available('tensorflow')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 104, in gpu_available
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     ext_base_name, available_fn, 'running with GPU', verbose) or False
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 90, in _check_extension_lambda
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     p.start()
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/multiprocessing/process.py", line 103, in start
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     'daemonic processes are not allowed to have children'
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || AssertionError: daemonic processes are not allowed to have children
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Extension tensorflow was NOT built with Gloo.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Checking whether extension torch was built with Gloo.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Traceback (most recent call last):
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 73, in _target_fn
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     ext = importlib.import_module('.' + ext_base_name, 'horovod')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/importlib/__init__.py", line 126, in import_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     return _bootstrap._gcd_import(name[level:], package, level)
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 994, in _gcd_import
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 971, in _find_and_load
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 955, in _find_and_load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 665, in _load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap_external>", line 678, in exec_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/torch/__init__.py", line 39, in <module>
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     from horovod.torch.mpi_ops import allreduce, allreduce_async, allreduce_, allreduce_async_
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 80, in <module>
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     _has_gpu = gpu_available('torch')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 104, in gpu_available
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     ext_base_name, available_fn, 'running with GPU', verbose) or False
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 90, in _check_extension_lambda
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     p.start()
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/multiprocessing/process.py", line 103, in start
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     'daemonic processes are not allowed to have children'
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || AssertionError: daemonic processes are not allowed to have children
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Extension torch was NOT built with Gloo.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Checking whether extension mxnet was built with Gloo.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Traceback (most recent call last):
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 73, in _target_fn
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     ext = importlib.import_module('.' + ext_base_name, 'horovod')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/importlib/__init__.py", line 126, in import_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     return _bootstrap._gcd_import(name[level:], package, level)
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 994, in _gcd_import
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 971, in _find_and_load
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 955, in _find_and_load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 665, in _load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap_external>", line 678, in exec_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/mxnet/__init__.py", line 23, in <module>
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     __file__, 'mpi_lib')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 51, in check_extension
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     'Horovod with %s=1 to debug the build error.' % (ext_name, ext_env_var))
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || ImportError: Extension horovod.mxnet has not been built.  If this is not expected, reinstall Horovod with HOROVOD_WITH_MXNET=1 to debug the build error.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Extension mxnet was NOT built with Gloo.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Traceback (most recent call last):
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/bin/horovodrun", line 21, in <module>
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     run_commandline()
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/run/run.py", line 860, in run_commandline
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     _run(args)
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/run/run.py", line 828, in _run
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     _launch_job(args, remote_host_names, settings, common_intfs, command)
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/run/run.py", line 850, in _launch_job
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     elif gloo_built(verbose=(settings.verbose >= 2)):
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 124, in gloo_built
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     raise RuntimeError('Failed to determine if Gloo support has been built. '
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || RuntimeError: Failed to determine if Gloo support has been built. Run again with --verbose for more details.

# Fixed code
def allreduce(
    tensor,
    average=None,
    device_dense="",
    device_sparse="",
    compression=Compression.none,
    op=None,
):
    """Perform an allreduce on a tf.Tensor or tf.IndexedSlices.

    This function performs a bandwidth-optimal ring allreduce on the input
    tensor. If the input is an tf.IndexedSlices, the function instead does an
    allgather on the values and the indices, effectively doing an allreduce on
    the represented tensor.

    Arguments:
        tensor: tf.Tensor, tf.Variable, or tf.IndexedSlices to reduce.
                The shape of the input must be identical across all ranks.
        average: DEPRECATED, please use op instead.
        device_dense: Device to be used for dense tensors. Uses GPU by default
                      if Horovod was built with HOROVOD_GPU_ALLREDUCE.
        device_sparse: Device to be used for sparse tensors. Uses GPU by default
                       if Horovod was built with HOROVOD_GPU_ALLGATHER.
        compression: Compression algorithm used to reduce the amount of data
                     sent and received by each worker node.  Defaults to not
                     using compression.
        op: The reduction operation to combine tensors across different ranks.
            Defaults to Average if None is given.

    Returns:
        A tensor of the same shape and type as `tensor`, summed across all
        processes.
    """
    op = handle_average_backwards_compatibility(op, average)
    # Averaging happens in framework code, so translate that to Sum for the actual call
    true_op = Sum if op == Average else op

    if isinstance(tensor, tf.IndexedSlices):
        # TODO: Need to fix this to actuall call Adasum
        if op == Adasum:
            raise NotImplementedError(
                "The Adasum reduction does not currently support sparse tensors. As a "
                "workaround please pass sparse_as_dense=True to DistributedOptimizer"
            )
        with tf.device(device_sparse):
            # For IndexedSlices, do two allgathers instead of an allreduce.
            horovod_size = tf.cast(size(), tensor.values.dtype)
            values = allgather(tensor.values)
            indices = allgather(tensor.indices)

            # To make this operation into an average, divide allgathered values by
            # the Horovod size.
            new_values = (values / horovod_size) if op == Average else values
        return tf.IndexedSlices(new_values, indices, dense_shape=tensor.dense_shape)
    else:
        with tf.device(device_dense):
            horovod_size = tf.cast(size(), dtype=tensor.dtype)
            tensor_compressed, ctx = compression.compress(tensor)
            summed_tensor_compressed = _allreduce(tensor_compressed, op=true_op)
            summed_tensor = compression.decompress(summed_tensor_compressed, ctx)
            if op == Adasum:
                if "CPU" not in tensor.device and gpu_available("tensorflow"):
                    if nccl_built():
                        if not is_homogeneous:
                            raise NotImplementedError(
                                "Running GPU Adasum on heterogeneous cluster is not supported yet."
                            )
                        elif not check_num_rank_power_of_2(int(size() / local_size())):
                            raise NotImplementedError(
                                "Running GPU Adasum with non-power of 2 nodes is not supported yet."
                            )
                        horovod_local_size = tf.cast(local_size(), dtype=tensor.dtype)
                        new_tensor = summed_tensor / horovod_local_size
                    else:
                        warnings.warn(
                            "Adasum reduction does not currently support GPU reduction using MPI. Tensors "
                            "are copied to CPU memory instead. To use Adasum for GPU reduction, please "
                            "compile Horovod with HOROVOD_GPU_ALLREDUCE=NCCL."
                        )
                        new_tensor = summed_tensor
                else:
                    if not check_num_rank_power_of_2(size()):
                        raise NotImplementedError(
                            "Running Adasum with non-power of 2 ranks is not supported yet."
                        )
                    new_tensor = summed_tensor
            else:
                new_tensor = (
                    (summed_tensor / horovod_size) if op == Average else summed_tensor
                )
        return new_tensor