Repository: horovod/horovod

Issue:
Hide third-party symbols to avoid conflicts
I was investigating a segfault with the following trace:
```
[ 0] /lib/x86_64-linux-gnu/libpthread.so.0(+0x11390)[0x7f57f51ff390]
[ 1] /HOME/.local/lib/python3.6/site-packages/horovod/common/mpi_lib.cpython-36m-x86_64-linux-gnu.so(_ZN11flatbuffers15vector_downward10reallocateEm+0x2f)[0x7f56fae87d0f]
[ 2] /HOME/.local/lib/python3.6/site-packages/pyarrow/libarrow.so.10(+0x1c9edb)[0x7f56f9716edb]
[ 3] /HOME/.local/lib/python3.6/site-packages/pyarrow/libarrow.so.10(_ZN5arrow3ipc8internal23WriteRecordBatchMessageEllRKSt6vectorINS1_13FieldMetadataESaIS3_EERKS2_INS1_14BufferMetadataESaIS8_EEPSt10shared_ptrINS_6BufferEE+0xd2)[0x7f56f97174a2]
[ 4] /HOME/.local/lib/python3.6/site-packages/pyarrow/libarrow.so.10(_ZN5arrow3ipc16WriteRecordBatchERKNS_11RecordBatchElPNS_2io12OutputStreamEPiPlPNS_10MemoryPoolEib+0x29b)[0x7f56f9729f8b]
[ 5] /HOME/.local/lib/python3.6/site-packages/pyarrow/libarrow.so.10(_ZN5arrow3ipc23RecordBatchStreamWriter16WriteRecordBatchERKNS_11RecordBatchEb+0xfb)[0x7f56f972a5eb]
[ 6] /HOME/.local/lib/python3.6/site-packages/pyarrow/libarrow.so.10(_ZN5arrow3ipc22WriteRecordBatchStreamERKSt6vectorISt10shared_ptrINS_11RecordBatchEESaIS4_EEPNS_2io12OutputStreamE+0x12a)[0x7f56f97284ba]
[ 7] /HOME/.local/lib/python3.6/site-packages/pyarrow/libarrow_python.so.10(_ZN5arrow2py18SerializedPyObject7WriteToEPNS_2io12OutputStreamE+0x126)[0x7f56f9315dc6]
[ 8] /HOME/.local/lib/python3.6/site-packages/pyarrow/lib.cpython-36m-x86_64-linux-gnu.so(+0xd6389)[0x7f56f9d22389]
[ 9] python(+0x10eb28)[0x55f96cae4b28]
...
```

It can be seen from the above stack trace, that pyarrow is using flatbuffers, and it calls into horovod for the function `_ZN11flatbuffers15vector_downward10reallocateEm`.

Since horovod is vendoring it's own version of flatbuffer, it has the potential to cause problems if mixed with a different version of flatbuffer. Although I'm not sure if this is the cause to my problems, but horovod should mark non-horovod symbols as `bind=LOCAL`, rather than `WEAK`, or `visibility=HIDDEN`, rather than `DEFAULT`:
```
$ readelf -Ws common/mpi_lib.cpython-36m-x86_64-linux-gnu.so |grep flatbuffer
    32: 000000000492ff90    48 OBJECT  WEAK   DEFAULT   24 _ZTVN11flatbuffers16simple_allocatorE  
   109: 0000000000020640    54 FUNC    WEAK   DEFAULT   11 _ZNSt8_Rb_treeIN11flatbuffers6OffsetINS0_6StringEEES3_St9_IdentityIS3_ENS0_17FlatBufferBuilder19StringOffsetCompareESaIS3_EE8_M_eraseEPSt13_Rb_tree_nodeIS3_E
   110: 000000000001ecd0     8 FUNC    WEAK   DEFAULT   11 _ZNK11flatbuffers16simple_allocator8allocateEm                                                                                           
   179: 00000000000164f0 10251 FUNC    GLOBAL DEFAULT   11 _ZN7horovod6common27MPIResponse_SerializeToWireERKNS0_11MPIResponseERN11flatbuffers17FlatBufferBuilderERNS4_6OffsetINS0_4wire11MPIResponseEEE
   195: 000000000001ec90     2 FUNC    WEAK   DEFAULT   11 _ZN11flatbuffers16simple_allocatorD1Ev 
   236: 0000000000020710   230 FUNC    WEAK   DEFAULT   11 _ZNSt6vectorIN11flatbuffers17FlatBufferBuilder8FieldLocESaIS2_EE19_M_emplace_back_auxIIRKS2_EEEvDpOT_                                    
   239: 000000000001ece0   185 FUNC    WEAK   DEFAULT   11 _ZN11flatbuffers15vector_downward10reallocateEm                                                                                          
   245: 0000000000020680   142 FUNC    WEAK   DEFAULT   11 _ZN11flatbuffers17FlatBufferBuilderD2Ev                                                                                                     254: 000000000001eca0     5 FUNC    WEAK   DEFAULT   11 _ZN11flatbuffers16simple_allocatorD0Ev 
   275: 00000000000789e0    34 OBJECT  WEAK   DEFAULT   13 _ZTSN11flatbuffers16simple_allocatorE  
   300: 0000000000020680   142 FUNC    WEAK   DEFAULT   11 _ZN11flatbuffers17FlatBufferBuilderD1Ev                                                                                                     306: 00000000000208d0  2308 FUNC    WEAK   DEFAULT   11 _ZN11flatbuffers17FlatBufferBuilder8EndTableEjt                                                                                          
   380: 000000000001ecb0    18 FUNC    WEAK   DEFAULT   11 _ZNK11flatbuffers16simple_allocator10deallocateEPh                                                                                       
   406: 000000000492ff80    16 OBJECT  WEAK   DEFAULT   24 _ZTIN11flatbuffers16simple_allocatorE  
   408: 0000000000020710   230 FUNC    WEAK   DEFAULT   11 _ZNSt6vectorIN11flatbuffers17FlatBufferBuilder8FieldLocESaIS2_EE19_M_emplace_back_auxIJRKS2_EEEvDpOT_                                    
   429: 000000000001ec90     2 FUNC    WEAK   DEFAULT   11 _ZN11flatbuffers16simple_allocatorD2Ev 
   468: 0000000004931b70     8 OBJECT  WEAK   DEFAULT   28 _ZN11flatbuffers25flatbuffer_version_stringE                                                                                             
    91: 000000000001a9d0  6375 FUNC    LOCAL  DEFAULT   11 _ZN7horovod6common12_GLOBAL__N_126MPIRequest_SerializeToWireERKNS0_10MPIRequestERN11flatbuffers17FlatBufferBuilderERNS5_6OffsetINS0_4wire10MPIRequestEEE.constprop.287
  1854: 000000000492ff90    48 OBJECT  WEAK   DEFAULT   24 _ZTVN11flatbuffers16simple_allocatorE  
  1931: 0000000000020640    54 FUNC    WEAK   DEFAULT   11 _ZNSt8_Rb_treeIN11flatbuffers6OffsetINS0_6StringEEES3_St9_IdentityIS3_ENS0_17FlatBufferBuilder19StringOffsetCompareESaIS3_EE8_M_eraseEPSt13_Rb_tree_nodeIS3_E
  1932: 000000000001ecd0     8 FUNC    WEAK   DEFAULT   11 _ZNK11flatbuffers16simple_allocator8allocateEm                                                                                           
  2001: 00000000000164f0 10251 FUNC    GLOBAL DEFAULT   11 _ZN7horovod6common27MPIResponse_SerializeToWireERKNS0_11MPIResponseERN11flatbuffers17FlatBufferBuilderERNS4_6OffsetINS0_4wire11MPIResponseEEE
  2017: 000000000001ec90     2 FUNC    WEAK   DEFAULT   11 _ZN11flatbuffers16simple_allocatorD1Ev 
  2058: 0000000000020710   230 FUNC    WEAK   DEFAULT   11 _ZNSt6vectorIN11flatbuffers17FlatBufferBuilder8FieldLocESaIS2_EE19_M_emplace_back_auxIIRKS2_EEEvDpOT_                                    
  2061: 000000000001ece0   185 FUNC    WEAK   DEFAULT   11 _ZN11flatbuffers15vector_downward10reallocateEm                                                                                          
  2067: 0000000000020680   142 FUNC    WEAK   DEFAULT   11 _ZN11flatbuffers17FlatBufferBuilderD2Ev                                                                                                  
  2076: 000000000001eca0     5 FUNC    WEAK   DEFAULT   11 _ZN11flatbuffers16simple_allocatorD0Ev 
  2097: 00000000000789e0    34 OBJECT  WEAK   DEFAULT   13 _ZTSN11flatbuffers16simple_allocatorE  
  2122: 0000000000020680   142 FUNC    WEAK   DEFAULT   11 _ZN11flatbuffers17FlatBufferBuilderD1Ev                                                                                                  
  2128: 00000000000208d0  2308 FUNC    WEAK   DEFAULT   11 _ZN11flatbuffers17FlatBufferBuilder8EndTableEjt                                                                                          
  2202: 000000000001ecb0    18 FUNC    WEAK   DEFAULT   11 _ZNK11flatbuffers16simple_allocator10deallocateEPh                                                                                       
  2228: 000000000492ff80    16 OBJECT  WEAK   DEFAULT   24 _ZTIN11flatbuffers16simple_allocatorE  
  2230: 0000000000020710   230 FUNC    WEAK   DEFAULT   11 _ZNSt6vectorIN11flatbuffers17FlatBufferBuilder8FieldLocESaIS2_EE19_M_emplace_back_auxIJRKS2_EEEvDpOT_                                    
  2251: 000000000001ec90     2 FUNC    WEAK   DEFAULT   11 _ZN11flatbuffers16simple_allocatorD2Ev 
  2290: 0000000004931b70     8 OBJECT  WEAK   DEFAULT   28 _ZN11flatbuffers25flatbuffer_version_stringE  
```



Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -74,7 +74,7 @@ def get_cpp_flags(build_ext):
         flags_to_try = [default_flags, default_flags + ['-stdlib=libc++']]
     for cpp_flags in flags_to_try:
         try:
-            test_compile(build_ext, 'test_cpp_flags', extra_preargs=cpp_flags,
+            test_compile(build_ext, 'test_cpp_flags', extra_compile_preargs=cpp_flags,
                          code=textwrap.dedent('''\
                     #include <unordered_map>
                     void test() {
@@ -91,6 +91,32 @@ def get_cpp_flags(build_ext):
     raise DistutilsPlatformError(last_err)
 
 
+def get_link_flags(build_ext):
+    last_err = None
+    libtool_flags = ['-Wl,-exported_symbols_list,horovod.exp']
+    ld_flags = ['-Wl,--version-script=horovod.lds']
+    if sys.platform == 'darwin':
+        flags_to_try = [libtool_flags, ld_flags]
+    else:
+        flags_to_try = [ld_flags, libtool_flags]
+    for link_flags in flags_to_try:
+        try:
+            test_compile(build_ext, 'test_link_flags', extra_link_preargs=link_flags,
+                         code=textwrap.dedent('''\
+                    void test() {
+                    }
+                    '''))
+
+            return link_flags
+        except (CompileError, LinkError):
+            last_err = 'Unable to determine C++ link flags (see error above).'
+        except Exception:
+            last_err = 'Unable to determine C++ link flags.  ' \
+                       'Last error:\n\n%s' % traceback.format_exc()
+
+    raise DistutilsPlatformError(last_err)
+
+
 def get_tf_include_dirs():
     import tensorflow as tf
     tf_inc = tf.sysconfig.get_include()
@@ -109,7 +135,7 @@ def get_tf_libs(build_ext, lib_dirs, cpp_flags):
         try:
             lib_file = test_compile(build_ext, 'test_tensorflow_libs',
                                     library_dirs=lib_dirs, libraries=tf_libs,
-                                    extra_preargs=cpp_flags,
+                                    extra_compile_preargs=cpp_flags,
                                     code=textwrap.dedent('''\
                     void test() {
                     }
@@ -136,7 +162,7 @@ def get_tf_abi(build_ext, include_dirs, lib_dirs, libs, cpp_flags):
             lib_file = test_compile(build_ext, 'test_tensorflow_abi',
                                     macros=[(cxx11_abi_macro, cxx11_abi)],
                                     include_dirs=include_dirs, library_dirs=lib_dirs,
-                                    libraries=libs, extra_preargs=cpp_flags,
+                                    libraries=libs, extra_compile_preargs=cpp_flags,
                                     code=textwrap.dedent('''\
                 #include <string>
                 #include "tensorflow/core/framework/op.h"
@@ -207,8 +233,8 @@ def get_mpi_flags():
             '%s' % (show_command, traceback.format_exc()))
 
 
-def test_compile(build_ext, name, code, libraries=None, include_dirs=None, library_dirs=None, macros=None,
-                 extra_preargs=None):
+def test_compile(build_ext, name, code, libraries=None, include_dirs=None, library_dirs=None,
+                 macros=None, extra_compile_preargs=None, extra_link_preargs=None):
     test_compile_dir = os.path.join(build_ext.build_temp, 'test_compile')
     if not os.path.exists(test_compile_dir):
         os.makedirs(test_compile_dir)
@@ -222,10 +248,11 @@ def test_compile(build_ext, name, code, libraries=None, include_dirs=None, libra
     shared_object_file = compiler.shared_object_filename(
         name, output_dir=test_compile_dir)
 
-    compiler.compile([source_file], extra_preargs=extra_preargs,
+    compiler.compile([source_file], extra_preargs=extra_compile_preargs,
                      include_dirs=include_dirs, macros=macros)
     compiler.link_shared_object(
-        [object_file], shared_object_file, libraries=libraries, library_dirs=library_dirs)
+        [object_file], shared_object_file, libraries=libraries, library_dirs=library_dirs,
+        extra_preargs=extra_link_preargs)
 
     return shared_object_file
 
@@ -254,7 +281,8 @@ def get_cuda_dirs(build_ext, cpp_flags):
 
     try:
         test_compile(build_ext, 'test_cuda', libraries=['cudart'], include_dirs=cuda_include_dirs,
-                     library_dirs=cuda_lib_dirs, extra_preargs=cpp_flags, code=textwrap.dedent('''\
+                     library_dirs=cuda_lib_dirs, extra_compile_preargs=cpp_flags,
+                     code=textwrap.dedent('''\
             #include <cuda_runtime.h>
             void test() {
                 cudaSetDevice(0);
@@ -299,7 +327,8 @@ def get_nccl_vals(build_ext, cuda_include_dirs, cuda_lib_dirs, cpp_flags):
 
     try:
         test_compile(build_ext, 'test_nccl', libraries=nccl_libs, include_dirs=nccl_include_dirs + cuda_include_dirs,
-                     library_dirs=nccl_lib_dirs + cuda_lib_dirs, extra_preargs=cpp_flags, code=textwrap.dedent('''\
+                     library_dirs=nccl_lib_dirs + cuda_lib_dirs, extra_compile_preargs=cpp_flags,
+                     code=textwrap.dedent('''\
             #include <nccl.h>
             #if NCCL_MAJOR < 2
             #error Horovod requires NCCL 2.0 or later version, please upgrade.
@@ -340,6 +369,7 @@ def get_ddl_dirs():
 
 def get_common_options(build_ext):
     cpp_flags = get_cpp_flags(build_ext)
+    link_flags = get_link_flags(build_ext)
     mpi_flags = get_mpi_flags()
 
     gpu_allreduce = os.environ.get('HOROVOD_GPU_ALLREDUCE')
@@ -393,7 +423,7 @@ def get_common_options(build_ext):
                'horovod/common/operations.cc',
                'horovod/common/timeline.cc']
     COMPILE_FLAGS = cpp_flags + shlex.split(mpi_flags)
-    LINK_FLAGS = shlex.split(mpi_flags)
+    LINK_FLAGS = link_flags + shlex.split(mpi_flags)
     LIBRARY_DIRS = []
     LIBRARIES = []
 
@@ -406,7 +436,6 @@ def get_common_options(build_ext):
     if have_nccl:
         MACROS += [('HAVE_NCCL', '1')]
         INCLUDES += nccl_include_dirs
-        LINK_FLAGS += ['-Wl,--version-script=hide_nccl.lds']
         LIBRARY_DIRS += nccl_lib_dirs
         LIBRARIES += nccl_libs
 
@@ -511,7 +540,7 @@ def is_torch_cuda_v2(build_ext, include_dirs, extra_compile_args):
     try:
         from torch.utils.cpp_extension import include_paths
         test_compile(build_ext, 'test_torch_cuda', include_dirs=include_dirs + include_paths(cuda=True),
-                     extra_preargs=extra_compile_args, code=textwrap.dedent('''\
+                     extra_compile_preargs=extra_compile_args, code=textwrap.dedent('''\
             #include <THC/THC.h>
             void test() {
             }


Test patch: