# Buggy code
def _allreduce_async(tensor, output, name, op):
    if tensor.dtype == torch.float16 and not _fp16_supported:
        raise NotImplementedError(
            "float16 allreduce is not supported for PyTorch version {} < 1.0.0".format(
                torch.__version__
            )
        )

    # Set the divisor for reduced gradients to average when necessary
    if op == Average:
        divisor = size()
    elif op == Adasum:
        if tensor.device.type != "cpu" and _has_gpu:
            if nccl_built():
                if not is_homogeneous():
                    raise NotImplementedError(
                        "Running GPU Adasum on heterogeneous cluster is not supported yet."
                    )
                elif not num_rank_is_power_2(int(size() / local_size())):
                    raise NotImplementedError(
                        "Running GPU Adasum with non-power of 2 nodes is not supported yet."
                    )
                divisor = local_size()
            else:
                warnings.warn(
                    "Adasum reduction does not currently support "
                    "GPU reduction using MPI. Tensors are copied to CPU memory instead."
                    "To use Adasum for GPU reduction, please compile Horovod with HOROVOD_GPU_ALLREDUCE=NCCL."
                )
                divisor = 1
        else:
            if not num_rank_is_power_2(size()):
                raise NotImplementedError(
                    "Running Adasum with non-power of 2 ranks is not supported yet."
                )
            divisor = 1
    else:
        divisor = 1
    # Averaging happens in framework code, so translate that to Sum for the actual call
    true_op = Sum if op == Average else op

    function = _check_function(_allreduce_function_factory, tensor)
    handle = getattr(mpi_lib, function)(
        tensor, output, divisor, name.encode() if name is not None else _NULL, true_op
    )
    _handle_map[handle] = (tensor, output)
    return handle


# Traceback (AssertionError)
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Checking whether extension tensorflow was built with MPI.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint8 = np.dtype([("qint8", np.int8, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_quint8 = np.dtype([("quint8", np.uint8, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint16 = np.dtype([("qint16", np.int16, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_quint16 = np.dtype([("quint16", np.uint16, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint32 = np.dtype([("qint32", np.int32, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   np_resource = np.dtype([("resource", np.ubyte, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint8 = np.dtype([("qint8", np.int8, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_quint8 = np.dtype([("quint8", np.uint8, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint16 = np.dtype([("qint16", np.int16, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_quint16 = np.dtype([("quint16", np.uint16, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint32 = np.dtype([("qint32", np.int32, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   np_resource = np.dtype([("resource", np.ubyte, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Traceback (most recent call last):
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 73, in _target_fn
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     ext = importlib.import_module('.' + ext_base_name, 'horovod')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/importlib/__init__.py", line 126, in import_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     return _bootstrap._gcd_import(name[level:], package, level)
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 994, in _gcd_import
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 971, in _find_and_load
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 955, in _find_and_load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 665, in _load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap_external>", line 678, in exec_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/tensorflow/__init__.py", line 43, in <module>
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     has_gpu = gpu_available('tensorflow')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 104, in gpu_available
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     ext_base_name, available_fn, 'running with GPU', verbose) or False
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 90, in _check_extension_lambda
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     p.start()
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/multiprocessing/process.py", line 103, in start
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     'daemonic processes are not allowed to have children'
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || AssertionError: daemonic processes are not allowed to have children
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Extension tensorflow was NOT built with MPI.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Checking whether extension torch was built with MPI.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Traceback (most recent call last):
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 73, in _target_fn
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     ext = importlib.import_module('.' + ext_base_name, 'horovod')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/importlib/__init__.py", line 126, in import_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     return _bootstrap._gcd_import(name[level:], package, level)
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 994, in _gcd_import
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 971, in _find_and_load
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 955, in _find_and_load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 665, in _load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap_external>", line 678, in exec_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/torch/__init__.py", line 39, in <module>
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     from horovod.torch.mpi_ops import allreduce, allreduce_async, allreduce_, allreduce_async_
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 80, in <module>
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     _has_gpu = gpu_available('torch')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 104, in gpu_available
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     ext_base_name, available_fn, 'running with GPU', verbose) or False
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 90, in _check_extension_lambda
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     p.start()
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/multiprocessing/process.py", line 103, in start
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     'daemonic processes are not allowed to have children'
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || AssertionError: daemonic processes are not allowed to have children
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Extension torch was NOT built with MPI.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Checking whether extension mxnet was built with MPI.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Traceback (most recent call last):
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 73, in _target_fn
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     ext = importlib.import_module('.' + ext_base_name, 'horovod')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/importlib/__init__.py", line 126, in import_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     return _bootstrap._gcd_import(name[level:], package, level)
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 994, in _gcd_import
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 971, in _find_and_load
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 955, in _find_and_load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 665, in _load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap_external>", line 678, in exec_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/mxnet/__init__.py", line 23, in <module>
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     __file__, 'mpi_lib')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 51, in check_extension
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     'Horovod with %s=1 to debug the build error.' % (ext_name, ext_env_var))
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Extension mxnet was NOT built with MPI.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || ImportError: Extension horovod.mxnet has not been built.  If this is not expected, reinstall Horovod with HOROVOD_WITH_MXNET=1 to debug the build error.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Checking whether extension tensorflow was built with Gloo.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint8 = np.dtype([("qint8", np.int8, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_quint8 = np.dtype([("quint8", np.uint8, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint16 = np.dtype([("qint16", np.int16, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_quint16 = np.dtype([("quint16", np.uint16, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint32 = np.dtype([("qint32", np.int32, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   np_resource = np.dtype([("resource", np.ubyte, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint8 = np.dtype([("qint8", np.int8, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_quint8 = np.dtype([("quint8", np.uint8, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint16 = np.dtype([("qint16", np.int16, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_quint16 = np.dtype([("quint16", np.uint16, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   _np_qint32 = np.dtype([("qint32", np.int32, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || /opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   np_resource = np.dtype([("resource", np.ubyte, 1)])
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Traceback (most recent call last):
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 73, in _target_fn
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     ext = importlib.import_module('.' + ext_base_name, 'horovod')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/importlib/__init__.py", line 126, in import_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     return _bootstrap._gcd_import(name[level:], package, level)
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 994, in _gcd_import
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 971, in _find_and_load
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 955, in _find_and_load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 665, in _load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap_external>", line 678, in exec_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/tensorflow/__init__.py", line 43, in <module>
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     has_gpu = gpu_available('tensorflow')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 104, in gpu_available
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     ext_base_name, available_fn, 'running with GPU', verbose) or False
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 90, in _check_extension_lambda
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     p.start()
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/multiprocessing/process.py", line 103, in start
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     'daemonic processes are not allowed to have children'
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || AssertionError: daemonic processes are not allowed to have children
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Extension tensorflow was NOT built with Gloo.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Checking whether extension torch was built with Gloo.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Traceback (most recent call last):
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 73, in _target_fn
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     ext = importlib.import_module('.' + ext_base_name, 'horovod')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/importlib/__init__.py", line 126, in import_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     return _bootstrap._gcd_import(name[level:], package, level)
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 994, in _gcd_import
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 971, in _find_and_load
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 955, in _find_and_load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 665, in _load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap_external>", line 678, in exec_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/torch/__init__.py", line 39, in <module>
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     from horovod.torch.mpi_ops import allreduce, allreduce_async, allreduce_, allreduce_async_
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 80, in <module>
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     _has_gpu = gpu_available('torch')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 104, in gpu_available
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     ext_base_name, available_fn, 'running with GPU', verbose) or False
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 90, in _check_extension_lambda
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     p.start()
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/multiprocessing/process.py", line 103, in start
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     'daemonic processes are not allowed to have children'
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || AssertionError: daemonic processes are not allowed to have children
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Extension torch was NOT built with Gloo.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Checking whether extension mxnet was built with Gloo.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Traceback (most recent call last):
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 73, in _target_fn
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     ext = importlib.import_module('.' + ext_base_name, 'horovod')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/importlib/__init__.py", line 126, in import_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     return _bootstrap._gcd_import(name[level:], package, level)
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 994, in _gcd_import
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 971, in _find_and_load
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 955, in _find_and_load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 665, in _load_unlocked
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap_external>", line 678, in exec_module
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/mxnet/__init__.py", line 23, in <module>
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     __file__, 'mpi_lib')
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 51, in check_extension
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     'Horovod with %s=1 to debug the build error.' % (ext_name, ext_env_var))
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || ImportError: Extension horovod.mxnet has not been built.  If this is not expected, reinstall Horovod with HOROVOD_WITH_MXNET=1 to debug the build error.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Extension mxnet was NOT built with Gloo.
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || Traceback (most recent call last):
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/bin/horovodrun", line 21, in <module>
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     run_commandline()
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/run/run.py", line 860, in run_commandline
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     _run(args)
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/run/run.py", line 828, in _run
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     _launch_job(args, remote_host_names, settings, common_intfs, command)
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/run/run.py", line 850, in _launch_job
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     elif gloo_built(verbose=(settings.verbose >= 2)):
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||   File "/opt/conda/lib/python3.6/site-packages/horovod/common/util.py", line 124, in gloo_built
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 ||     raise RuntimeError('Failed to determine if Gloo support has been built. '
d996a8f8-e1d4-4e1e-8d9e-e3c5804c0369 || RuntimeError: Failed to determine if Gloo support has been built. Run again with --verbose for more details.

# Fixed code
def _allreduce_async(tensor, output, name, op):
    if tensor.dtype == torch.float16 and not _fp16_supported:
        raise NotImplementedError(
            "float16 allreduce is not supported for PyTorch version {} < 1.0.0".format(
                torch.__version__
            )
        )

    # Set the divisor for reduced gradients to average when necessary
    if op == Average:
        divisor = size()
    elif op == Adasum:
        if tensor.device.type != "cpu" and gpu_available("torch"):
            if nccl_built():
                if not is_homogeneous():
                    raise NotImplementedError(
                        "Running GPU Adasum on heterogeneous cluster is not supported yet."
                    )
                elif not num_rank_is_power_2(int(size() / local_size())):
                    raise NotImplementedError(
                        "Running GPU Adasum with non-power of 2 nodes is not supported yet."
                    )
                divisor = local_size()
            else:
                warnings.warn(
                    "Adasum reduction does not currently support GPU reduction using MPI. Tensors are "
                    "copied to CPU memory instead. To use Adasum for GPU reduction, please compile Horovod "
                    "with HOROVOD_GPU_ALLREDUCE=NCCL."
                )
                divisor = 1
        else:
            if not num_rank_is_power_2(size()):
                raise NotImplementedError(
                    "Running Adasum with non-power of 2 ranks is not supported yet."
                )
            divisor = 1
    else:
        divisor = 1
    # Averaging happens in framework code, so translate that to Sum for the actual call
    true_op = Sum if op == Average else op

    function = _check_function(_allreduce_function_factory, tensor)
    handle = getattr(mpi_lib, function)(
        tensor, output, divisor, name.encode() if name is not None else _NULL, true_op
    )
    _handle_map[handle] = (tensor, output)
    return handle