Repository: horovod/horovod

Issue:
Problem to build horovod (horovod/torch/tensor_util.cc:39:30: error: ‘THByteTensor_newSizeOf’ was not declared in this scope)
Hello. I was trying to build the Horovod library and I got this error when compiling the Horovod with the pytorch from the master brach of its repository. However I was able to compile the Horovod using pytorch 0.4.1, Is there any workaround besides working with the 0.4.1 ?
thanks
```bash
tensor_util.cc: In static member function ‘static void horovod::torch::TensorUtil::CopyCPUToCuda(T*, TC*) [with horovod::common::MPIDataType DT = (horovod::common::MPIDataType)0; T = THTensor; TC = THTensor]’:
/home/disk0/.pytorch/horovod/horovod/torch/tensor_util.cc:39:30: error: ‘THByteTensor_newSizeOf’ was not declared in this scope
                              THByteTensor, THCudaByteStorage)

/home/disk0/.pytorch/horovod/horovod/torch/tensor_util.cc:38:58: error: ‘THCudaByteTensor_isSize’ was not declared in this scope

/home/disk0/.pytorch/horovod/horovod/torch/tensor_util.cc:39:30: error: ‘THByteTensor_isSize’ was not declared in this scope
                              THByteTensor, THCudaByteStorage)
 TENSOR_UTIL_DEFINE_CUDA_TYPE(MPIDataType::HOROVOD_UINT8, THCudaByteTensor,
```



Discussion:
@labor00, we will look into this breaking change.  The change was introduced in [this commit](https://github.com/pytorch/pytorch/commit/6ac35b35d12b65851f3269b838dc36d64e97237b), so pinning your repo to a revision prior to that should work.

Resolving patch:
diff --git a/horovod/torch/__init__.py b/horovod/torch/__init__.py
--- a/horovod/torch/__init__.py
+++ b/horovod/torch/__init__.py
@@ -19,8 +19,12 @@
 
 from horovod.common import check_extension
 
-check_extension('horovod.torch', 'HOROVOD_WITH_PYTORCH',
-                __file__, 'mpi_lib', '_mpi_lib')
+try:
+    check_extension('horovod.torch', 'HOROVOD_WITH_PYTORCH',
+                    __file__, 'mpi_lib_v2')
+except:
+    check_extension('horovod.torch', 'HOROVOD_WITH_PYTORCH',
+                    __file__, 'mpi_lib', '_mpi_lib')
 
 from horovod.torch.mpi_ops import allreduce, allreduce_async, allreduce_, allreduce_async_
 from horovod.torch.mpi_ops import allgather, allgather_async
diff --git a/horovod/torch/mpi_ops.py b/horovod/torch/mpi_ops.py
--- a/horovod/torch/mpi_ops.py
+++ b/horovod/torch/mpi_ops.py
@@ -20,11 +20,17 @@
 # Load all the necessary PyTorch C types.
 import torch
 
-from horovod.torch import mpi_lib_impl
-from horovod.torch import mpi_lib
-from horovod.common import HorovodBasics as _HorovodBasics
-
-_basics = _HorovodBasics(__file__, 'mpi_lib_impl', '_mpi_lib_impl')
+try:
+    from horovod.torch import mpi_lib_v2 as mpi_lib
+    from horovod.common import HorovodBasics as _HorovodBasics
+    _NULL = ""
+    _basics = _HorovodBasics(__file__, 'mpi_lib_v2')
+except:
+    from horovod.torch import mpi_lib_impl
+    from horovod.torch import mpi_lib
+    from horovod.common import HorovodBasics as _HorovodBasics
+    _NULL = mpi_lib._ffi.NULL
+    _basics = _HorovodBasics(__file__, 'mpi_lib_impl', '_mpi_lib_impl')
 
 # import basic methods
 init = _basics.init
@@ -42,10 +48,6 @@
 _handle_map = {}
 
 
-# Null pointer.
-_NULL = mpi_lib._ffi.NULL
-
-
 def _check_function(function_factory, tensor):
     function = function_factory(tensor)
     if not hasattr(mpi_lib, function):
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -31,6 +31,7 @@
 tensorflow_mpi_lib = Extension('horovod.tensorflow.mpi_lib', [])
 torch_mpi_lib = Extension('horovod.torch.mpi_lib', [])
 torch_mpi_lib_impl = Extension('horovod.torch.mpi_lib_impl', [])
+torch_mpi_lib_v2 = Extension('horovod.torch.mpi_lib_v2', [])
 
 
 def is_build_action():
@@ -506,6 +507,21 @@ def is_torch_cuda():
         return False
 
 
+def is_torch_cuda_v2(build_ext, include_dirs, extra_compile_args):
+    try:
+        from torch.utils.cpp_extension import include_paths
+        test_compile(build_ext, 'test_torch_cuda', include_dirs=include_dirs + include_paths(cuda=True),
+                     extra_preargs=extra_compile_args, code=textwrap.dedent('''\
+            #include <THC/THC.h>
+            void test() {
+            }
+            '''))
+        return True
+    except (CompileError, LinkError):
+        print('INFO: Above error indicates that this PyTorch installation does not support CUDA.')
+        return False
+
+
 def check_macro(macros, key):
     return any(k == key and v for k, v in macros)
 
@@ -530,9 +546,7 @@ def __exit__(self, type, value, traceback):
             os.rename(file + '.protected', file)
 
 
-def build_torch_extension(build_ext, options):
-    torch_version = check_torch_version()
-
+def build_torch_extension(build_ext, options, torch_version):
     have_cuda = is_torch_cuda()
     if not have_cuda and check_macro(options['MACROS'], 'HAVE_CUDA'):
         raise DistutilsPlatformError(
@@ -593,6 +607,53 @@ def build_torch_extension(build_ext, options):
         build_ext.build_extension(setuptools_ext)
 
 
+def build_torch_extension_v2(build_ext, options, torch_version):
+    have_cuda = is_torch_cuda_v2(build_ext, include_dirs=options['INCLUDES'],
+                                 extra_compile_args=options['COMPILE_FLAGS'])
+    if not have_cuda and check_macro(options['MACROS'], 'HAVE_CUDA'):
+        raise DistutilsPlatformError(
+            'Horovod build with GPU support was requested, but this PyTorch '
+            'installation does not support CUDA.')
+
+    # Update HAVE_CUDA to mean that PyTorch supports CUDA. Internally, we will be checking
+    # HOROVOD_GPU_(ALLREDUCE|ALLGATHER|BROADCAST) to decide whether we should use GPU
+    # version or transfer tensors to CPU memory for those operations.
+    updated_macros = set_macro(
+        options['MACROS'], 'HAVE_CUDA', str(int(have_cuda)))
+
+    # Export TORCH_VERSION equal to our representation of torch.__version__. Internally it's
+    # used for backwards compatibility checks.
+    updated_macros = set_macro(updated_macros, 'TORCH_VERSION', str(torch_version))
+
+    # Always set _GLIBCXX_USE_CXX11_ABI, since PyTorch can only detect whether it was set to 1.
+    import torch
+    updated_macros = set_macro(updated_macros, '_GLIBCXX_USE_CXX11_ABI',
+                               str(int(torch.compiled_with_cxx11_abi())))
+
+    if have_cuda:
+        from torch.utils.cpp_extension import CUDAExtension as TorchExtension
+    else:
+        # CUDAExtension fails with `ld: library not found for -lcudart` if CUDA is not present
+        from torch.utils.cpp_extension import CppExtension as TorchExtension
+    ext = TorchExtension(torch_mpi_lib_v2.name,
+                         define_macros=updated_macros,
+                         include_dirs=options['INCLUDES'],
+                         sources=options['SOURCES'] + ['horovod/torch/mpi_ops_v2.cc',
+                                                       'horovod/torch/handle_manager.cc',
+                                                       'horovod/torch/ready_event.cc',
+                                                       'horovod/torch/cuda_util.cc',
+                                                       'horovod/torch/adapter_v2.cc'],
+                         extra_compile_args=options['COMPILE_FLAGS'],
+                         extra_link_args=options['LINK_FLAGS'],
+                         library_dirs=options['LIBRARY_DIRS'],
+                         libraries=options['LIBRARIES'])
+
+    # Patch an existing torch_mpi_lib_v2 extension object.
+    for k, v in ext.__dict__.items():
+        torch_mpi_lib_v2.__dict__[k] = v
+    build_ext.build_extension(torch_mpi_lib_v2)
+
+
 # run the customize_compiler
 class custom_build_ext(build_ext):
     def build_extensions(self):
@@ -614,7 +675,11 @@ def build_extensions(self):
                     raise
         if not os.environ.get('HOROVOD_WITHOUT_PYTORCH'):
             try:
-                build_torch_extension(self, options)
+                torch_version = check_torch_version()
+                if torch_version >= 4002000:
+                    build_torch_extension_v2(self, options, torch_version)
+                else:
+                    build_torch_extension(self, options, torch_version)
                 built_plugins.append(True)
             except:
                 if not os.environ.get('HOROVOD_WITH_PYTORCH'):
@@ -643,7 +708,7 @@ def build_extensions(self):
       classifiers=[
           'License :: OSI Approved :: Apache Software License'
       ],
-      ext_modules=[tensorflow_mpi_lib, torch_mpi_lib, torch_mpi_lib_impl],
+      ext_modules=[tensorflow_mpi_lib, torch_mpi_lib, torch_mpi_lib_impl, torch_mpi_lib_v2],
       cmdclass={'build_ext': custom_build_ext},
       # cffi is required for PyTorch
       # If cffi is specified in setup_requires, it will need libffi to be installed on the machine,


Test patch:
diff --git a/test/test_torch.py b/test/test_torch.py
--- a/test/test_torch.py
+++ b/test/test_torch.py
@@ -244,7 +244,7 @@ def test_horovod_allreduce_error(self):
         try:
             hvd.allreduce(tensor)
             assert False, 'hvd.allreduce did not throw error'
-        except torch.FatalError:
+        except (torch.FatalError, RuntimeError):
             pass
 
         # Same number of elements, different rank
@@ -257,7 +257,7 @@ def test_horovod_allreduce_error(self):
         try:
             hvd.allreduce(tensor)
             assert False, 'hvd.allreduce did not throw error'
-        except torch.FatalError:
+        except (torch.FatalError, RuntimeError):
             pass
 
     def test_horovod_allreduce_type_error(self):
@@ -281,7 +281,7 @@ def test_horovod_allreduce_type_error(self):
         try:
             hvd.allreduce(tensor)
             assert False, 'hvd.allreduce did not throw error'
-        except torch.FatalError:
+        except (torch.FatalError, RuntimeError):
             pass
 
     def test_horovod_allreduce_cpu_gpu_error(self):
@@ -309,7 +309,7 @@ def test_horovod_allreduce_cpu_gpu_error(self):
         try:
             hvd.allreduce(tensor)
             assert False, 'hvd.allreduce did not throw error'
-        except torch.FatalError:
+        except (torch.FatalError, RuntimeError):
             pass
 
     def test_horovod_allreduce_grad(self):
@@ -444,7 +444,7 @@ def test_horovod_allgather_error(self):
         try:
             hvd.allgather(tensor)
             assert False, 'hvd.allgather did not throw error'
-        except torch.FatalError:
+        except (torch.FatalError, RuntimeError):
             pass
 
     def test_horovod_allgather_type_error(self):
@@ -467,7 +467,7 @@ def test_horovod_allgather_type_error(self):
         try:
             hvd.allgather(tensor)
             assert False, 'hvd.allgather did not throw error'
-        except torch.FatalError:
+        except (torch.FatalError, RuntimeError):
             pass
 
     def test_horovod_allgather_grad(self):
@@ -588,7 +588,7 @@ def test_horovod_broadcast_error(self):
         try:
             hvd.broadcast(tensor, 0)
             assert False, 'hvd.broadcast did not throw error'
-        except torch.FatalError:
+        except (torch.FatalError, RuntimeError):
             pass
 
     def test_horovod_broadcast_type_error(self):
@@ -611,7 +611,7 @@ def test_horovod_broadcast_type_error(self):
         try:
             hvd.broadcast(tensor, 0)
             assert False, 'hvd.broadcast did not throw error'
-        except torch.FatalError:
+        except (torch.FatalError, RuntimeError):
             pass
 
     def test_horovod_broadcast_rank_error(self):
@@ -630,7 +630,7 @@ def test_horovod_broadcast_rank_error(self):
         try:
             hvd.broadcast(tensor, rank)
             assert False, 'hvd.broadcast did not throw error'
-        except torch.FatalError:
+        except (torch.FatalError, RuntimeError):
             pass
 
     def test_horovod_broadcast_grad(self):