Repository: horovod/horovod

Issue:
pip install horovod fails with error: Unable to determine CXX11 ABI to use with TensorFlow
Hi,

sasikanthkotti$ pip install horovod on mac fails as below : 

Collecting horovod
  Using cached horovod-0.11.0.tar.gz
Building wheels for collected packages: horovod
  Running setup.py bdist_wheel for horovod ... error
  Complete output from command /anaconda2/envs/Tensorflow_Python3_6/bin/python -u -c "import setuptools, tokenize;__file__='/private/var/folders/24/sln_smf506j41c593wqcmm180000gn/T/pip-build-l5wzzk3m/horovod/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\r\n', '\n');f.close();exec(compile(code, __file__, 'exec'))" bdist_wheel -d /var/folders/24/sln_smf506j41c593wqcmm180000gn/T/tmpuadcilgepip-wheel- --python-tag cp36:
  running bdist_wheel
  running build
  running build_py
  creating build
  creating build/lib.macosx-10.7-x86_64-3.6
  creating build/lib.macosx-10.7-x86_64-3.6/horovod
  copying horovod/__init__.py -> build/lib.macosx-10.7-x86_64-3.6/horovod
  creating build/lib.macosx-10.7-x86_64-3.6/horovod/keras
  copying horovod/keras/__init__.py -> build/lib.macosx-10.7-x86_64-3.6/horovod/keras
  copying horovod/keras/callbacks.py -> build/lib.macosx-10.7-x86_64-3.6/horovod/keras
  creating build/lib.macosx-10.7-x86_64-3.6/horovod/tensorflow
  copying horovod/tensorflow/mpi_ops.py -> build/lib.macosx-10.7-x86_64-3.6/horovod/tensorflow
  copying horovod/tensorflow/__init__.py -> build/lib.macosx-10.7-x86_64-3.6/horovod/tensorflow
  copying horovod/tensorflow/mpi_ops_test.py -> build/lib.macosx-10.7-x86_64-3.6/horovod/tensorflow
  running build_ext
  gcc -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -I/anaconda2/envs/Tensorflow_Python3_6/include -arch x86_64 -I/anaconda2/envs/Tensorflow_Python3_6/include -arch x86_64 -std=c++11 -I/anaconda2/envs/Tensorflow_Python3_6/include/python3.6m -c build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_libs.cc -o build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_libs.o
  gcc -bundle -undefined dynamic_lookup -L/anaconda2/envs/Tensorflow_Python3_6/lib -arch x86_64 -L/anaconda2/envs/Tensorflow_Python3_6/lib -arch x86_64 -arch x86_64 build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_libs.o -L/anaconda2/envs/Tensorflow_Python3_6/lib/python3.6/site-packages/tensorflow/core -L/anaconda2/envs/Tensorflow_Python3_6/lib -ltensorflow_framework -o build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_libs.so
  ld: library not found for -ltensorflow_framework
  clang: error: linker command failed with exit code 1 (use -v to see invocation)
  gcc -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -I/anaconda2/envs/Tensorflow_Python3_6/include -arch x86_64 -I/anaconda2/envs/Tensorflow_Python3_6/include -arch x86_64 -std=c++11 -I/anaconda2/envs/Tensorflow_Python3_6/include/python3.6m -c build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_libs.cc -o build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_libs.o
  gcc -bundle -undefined dynamic_lookup -L/anaconda2/envs/Tensorflow_Python3_6/lib -arch x86_64 -L/anaconda2/envs/Tensorflow_Python3_6/lib -arch x86_64 -arch x86_64 build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_libs.o -L/anaconda2/envs/Tensorflow_Python3_6/lib/python3.6/site-packages/tensorflow/core -L/anaconda2/envs/Tensorflow_Python3_6/lib -o build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_libs.so
  gcc -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -I/anaconda2/envs/Tensorflow_Python3_6/include -arch x86_64 -I/anaconda2/envs/Tensorflow_Python3_6/include -arch x86_64 -std=c++11 -D_GLIBCXX_USE_CXX11_ABI=0 -I/anaconda2/envs/Tensorflow_Python3_6/lib/python3.6/site-packages/tensorflow/include -I/anaconda2/envs/Tensorflow_Python3_6/lib/python3.6/site-packages/tensorflow/include/external/nsync/public -I/anaconda2/envs/Tensorflow_Python3_6/include/python3.6m -c build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_abi.cc -o build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_abi.o
  In file included from build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_abi.cc:2:
  /anaconda2/envs/Tensorflow_Python3_6/lib/python3.6/site-packages/tensorflow/include/tensorflow/core/framework/op.h:20:10: fatal error: 'unordered_map' file not found
  #include <unordered_map>
           ^~~~~~~~~~~~~~~
  1 error generated.
  gcc -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -I/anaconda2/envs/Tensorflow_Python3_6/include -arch x86_64 -I/anaconda2/envs/Tensorflow_Python3_6/include -arch x86_64 -std=c++11 -D_GLIBCXX_USE_CXX11_ABI=1 -I/anaconda2/envs/Tensorflow_Python3_6/lib/python3.6/site-packages/tensorflow/include -I/anaconda2/envs/Tensorflow_Python3_6/lib/python3.6/site-packages/tensorflow/include/external/nsync/public -I/anaconda2/envs/Tensorflow_Python3_6/include/python3.6m -c build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_abi.cc -o build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_abi.o
  In file included from build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_abi.cc:2:
  /anaconda2/envs/Tensorflow_Python3_6/lib/python3.6/site-packages/tensorflow/include/tensorflow/core/framework/op.h:20:10: fatal error: 'unordered_map' file not found
  #include <unordered_map>
           ^~~~~~~~~~~~~~~
  1 error generated.
  error: Unable to determine CXX11 ABI to use with TensorFlow (see error above).

  ----------------------------------------
  Failed building wheel for horovod
  Running setup.py clean for horovod
Failed to build horovod
Installing collected packages: horovod
  Running setup.py install for horovod ... error
    Complete output from command /anaconda2/envs/Tensorflow_Python3_6/bin/python -u -c "import setuptools, tokenize;__file__='/private/var/folders/24/sln_smf506j41c593wqcmm180000gn/T/pip-build-l5wzzk3m/horovod/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\r\n', '\n');f.close();exec(compile(code, __file__, 'exec'))" install --record /var/folders/24/sln_smf506j41c593wqcmm180000gn/T/pip-7s3cvoih-record/install-record.txt --single-version-externally-managed --compile:
    running install
    running build
    running build_py
    creating build
    creating build/lib.macosx-10.7-x86_64-3.6
    creating build/lib.macosx-10.7-x86_64-3.6/horovod
    copying horovod/__init__.py -> build/lib.macosx-10.7-x86_64-3.6/horovod
    creating build/lib.macosx-10.7-x86_64-3.6/horovod/keras
    copying horovod/keras/__init__.py -> build/lib.macosx-10.7-x86_64-3.6/horovod/keras
    copying horovod/keras/callbacks.py -> build/lib.macosx-10.7-x86_64-3.6/horovod/keras
    creating build/lib.macosx-10.7-x86_64-3.6/horovod/tensorflow
    copying horovod/tensorflow/mpi_ops.py -> build/lib.macosx-10.7-x86_64-3.6/horovod/tensorflow
    copying horovod/tensorflow/__init__.py -> build/lib.macosx-10.7-x86_64-3.6/horovod/tensorflow
    copying horovod/tensorflow/mpi_ops_test.py -> build/lib.macosx-10.7-x86_64-3.6/horovod/tensorflow
    running build_ext
    gcc -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -I/anaconda2/envs/Tensorflow_Python3_6/include -arch x86_64 -I/anaconda2/envs/Tensorflow_Python3_6/include -arch x86_64 -std=c++11 -I/anaconda2/envs/Tensorflow_Python3_6/include/python3.6m -c build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_libs.cc -o build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_libs.o
    gcc -bundle -undefined dynamic_lookup -L/anaconda2/envs/Tensorflow_Python3_6/lib -arch x86_64 -L/anaconda2/envs/Tensorflow_Python3_6/lib -arch x86_64 -arch x86_64 build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_libs.o -L/anaconda2/envs/Tensorflow_Python3_6/lib/python3.6/site-packages/tensorflow/core -L/anaconda2/envs/Tensorflow_Python3_6/lib -ltensorflow_framework -o build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_libs.so
    ld: library not found for -ltensorflow_framework
    clang: error: linker command failed with exit code 1 (use -v to see invocation)
    gcc -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -I/anaconda2/envs/Tensorflow_Python3_6/include -arch x86_64 -I/anaconda2/envs/Tensorflow_Python3_6/include -arch x86_64 -std=c++11 -I/anaconda2/envs/Tensorflow_Python3_6/include/python3.6m -c build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_libs.cc -o build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_libs.o
    gcc -bundle -undefined dynamic_lookup -L/anaconda2/envs/Tensorflow_Python3_6/lib -arch x86_64 -L/anaconda2/envs/Tensorflow_Python3_6/lib -arch x86_64 -arch x86_64 build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_libs.o -L/anaconda2/envs/Tensorflow_Python3_6/lib/python3.6/site-packages/tensorflow/core -L/anaconda2/envs/Tensorflow_Python3_6/lib -o build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_libs.so
    gcc -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -I/anaconda2/envs/Tensorflow_Python3_6/include -arch x86_64 -I/anaconda2/envs/Tensorflow_Python3_6/include -arch x86_64 -std=c++11 -D_GLIBCXX_USE_CXX11_ABI=0 -I/anaconda2/envs/Tensorflow_Python3_6/lib/python3.6/site-packages/tensorflow/include -I/anaconda2/envs/Tensorflow_Python3_6/lib/python3.6/site-packages/tensorflow/include/external/nsync/public -I/anaconda2/envs/Tensorflow_Python3_6/include/python3.6m -c build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_abi.cc -o build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_abi.o
    In file included from build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_abi.cc:2:
    /anaconda2/envs/Tensorflow_Python3_6/lib/python3.6/site-packages/tensorflow/include/tensorflow/core/framework/op.h:20:10: fatal error: 'unordered_map' file not found
    #include <unordered_map>
             ^~~~~~~~~~~~~~~
    1 error generated.
    gcc -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -I/anaconda2/envs/Tensorflow_Python3_6/include -arch x86_64 -I/anaconda2/envs/Tensorflow_Python3_6/include -arch x86_64 -std=c++11 -D_GLIBCXX_USE_CXX11_ABI=1 -I/anaconda2/envs/Tensorflow_Python3_6/lib/python3.6/site-packages/tensorflow/include -I/anaconda2/envs/Tensorflow_Python3_6/lib/python3.6/site-packages/tensorflow/include/external/nsync/public -I/anaconda2/envs/Tensorflow_Python3_6/include/python3.6m -c build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_abi.cc -o build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_abi.o
    In file included from build/temp.macosx-10.7-x86_64-3.6/test_compile/test_tensorflow_abi.cc:2:
    /anaconda2/envs/Tensorflow_Python3_6/lib/python3.6/site-packages/tensorflow/include/tensorflow/core/framework/op.h:20:10: fatal error: 'unordered_map' file not found
    #include <unordered_map>
             ^~~~~~~~~~~~~~~
    1 error generated.
    error: Unable to determine CXX11 ABI to use with TensorFlow (see error above).

    ----------------------------------------
Command "/anaconda2/envs/Tensorflow_Python3_6/bin/python -u -c "import setuptools, tokenize;__file__='/private/var/folders/24/sln_smf506j41c593wqcmm180000gn/T/pip-build-l5wzzk3m/horovod/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\r\n', '\n');f.close();exec(compile(code, __file__, 'exec'))" install --record /var/folders/24/sln_smf506j41c593wqcmm180000gn/T/pip-7s3cvoih-record/install-record.txt --single-version-externally-managed --compile" failed with error code 1 in /private/var/folders/24/sln_smf506j41c593wqcmm180000gn/T/pip-build-l5wzzk3m/horovod/

Details : 
OS : macOS High Sierra Version 10.13.1
Open MPI: 3.0.0

Please advice.

Thanks and Regards,
Sasikanth


Discussion:
@ksasi, could you try to add `CFLAGS=-stdlib=libc++` in front of your `pip install horovod`?
Adding the flag CFLAGS=-stdlib=libc++ in front of your pip install horovod, seems to have helped. Below is the output : 

CFLAGS=-stdlib=libc++ pip install horovod
Collecting horovod
  Using cached horovod-0.11.0.tar.gz
Building wheels for collected packages: horovod
  Running setup.py bdist_wheel for horovod ... done
  Stored in directory: /Users/sasikanthkotti/Library/Caches/pip/wheels/20/1e/24/2e9619fed60caccab50730f226c9972f503090efc2adb231bc
Successfully built horovod
Installing collected packages: horovod
Successfully installed horovod-0.11.0

Thanks and Regards,
Sasikanth

Resolving patch:
diff --git a/horovod/__init__.py b/horovod/__init__.py
--- a/horovod/__init__.py
+++ b/horovod/__init__.py
@@ -1 +1 @@
-__version__ = '0.11.0'
+__version__ = '0.11.1'
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -18,6 +18,7 @@
 from setuptools.command.build_ext import build_ext
 import shlex
 import subprocess
+import sys
 import textwrap
 import traceback
 
@@ -43,6 +44,33 @@ def check_tf_version():
             'Your TensorFlow version is outdated.  Horovod requires tensorflow>=1.1.0')
 
 
+def get_cpp_flags(build_ext):
+    last_err = None
+    default_flags = ['-std=c++11', '-fPIC', '-O2']
+    if sys.platform == 'darwin':
+        # Darwin most likely will have Clang, which has libc++.
+        flags_to_try = [default_flags + ['-stdlib=libc++'], default_flags]
+    else:
+        flags_to_try = [default_flags, default_flags + ['-stdlib=libc++']]
+    for cpp_flags in flags_to_try:
+        try:
+            test_compile(build_ext, 'test_cpp_flags', extra_preargs=cpp_flags,
+                         code=textwrap.dedent('''\
+                    #include <unordered_map>
+                    void test() {
+                    }
+                    '''))
+
+            return cpp_flags
+        except (CompileError, LinkError):
+            last_err = 'Unable to determine C++ compilation flags (see error above).'
+        except Exception:
+            last_err = 'Unable to determine C++ compilation flags.  ' \
+                       'Last error:\n\n%s' % traceback.format_exc()
+
+    raise DistutilsPlatformError(last_err)
+
+
 def get_tf_include_dirs():
     import tensorflow as tf
     tf_inc = tf.sysconfig.get_include()
@@ -55,12 +83,13 @@ def get_tf_lib_dirs():
     return [tf_lib]
 
 
-def get_tf_libs(build_ext, lib_dirs):
+def get_tf_libs(build_ext, lib_dirs, cpp_flags):
     last_err = None
     for tf_libs in [['tensorflow_framework'], []]:
         try:
             lib_file = test_compile(build_ext, 'test_tensorflow_libs',
                                     library_dirs=lib_dirs, libraries=tf_libs,
+                                    extra_preargs=cpp_flags,
                                     code=textwrap.dedent('''\
                     void test() {
                     }
@@ -79,7 +108,7 @@ def get_tf_libs(build_ext, lib_dirs):
     raise DistutilsPlatformError(last_err)
 
 
-def get_tf_abi(build_ext, include_dirs, lib_dirs, libs):
+def get_tf_abi(build_ext, include_dirs, lib_dirs, libs, cpp_flags):
     last_err = None
     cxx11_abi_macro = '_GLIBCXX_USE_CXX11_ABI'
     for cxx11_abi in ['0', '1']:
@@ -87,7 +116,8 @@ def get_tf_abi(build_ext, include_dirs, lib_dirs, libs):
             lib_file = test_compile(build_ext, 'test_tensorflow_abi',
                                     macros=[(cxx11_abi_macro, cxx11_abi)],
                                     include_dirs=include_dirs, library_dirs=lib_dirs,
-                                    libraries=libs, code=textwrap.dedent('''\
+                                    libraries=libs, extra_preargs=cpp_flags,
+                                    code=textwrap.dedent('''\
                 #include <string>
                 #include "tensorflow/core/framework/op.h"
                 #include "tensorflow/core/framework/op_kernel.h"
@@ -110,7 +140,7 @@ def get_tf_abi(build_ext, include_dirs, lib_dirs, libs):
     raise DistutilsPlatformError(last_err)
 
 
-def get_tf_flags(build_ext):
+def get_tf_flags(build_ext, cpp_flags):
     import tensorflow as tf
     try:
         return tf.sysconfig.get_compile_flags(), tf.sysconfig.get_link_flags()
@@ -118,8 +148,8 @@ def get_tf_flags(build_ext):
         # fallback to the previous logic
         tf_include_dirs = get_tf_include_dirs()
         tf_lib_dirs = get_tf_lib_dirs()
-        tf_libs = get_tf_libs(build_ext, tf_lib_dirs)
-        tf_abi = get_tf_abi(build_ext, tf_include_dirs, tf_lib_dirs, tf_libs)
+        tf_libs = get_tf_libs(build_ext, tf_lib_dirs, cpp_flags)
+        tf_abi = get_tf_abi(build_ext, tf_include_dirs, tf_lib_dirs, tf_libs, cpp_flags)
 
         compile_flags = []
         for include_dir in tf_include_dirs:
@@ -156,7 +186,8 @@ def get_mpi_flags():
             '%s' % (show_command, traceback.format_exc()))
 
 
-def test_compile(build_ext, name, code, libraries=None, include_dirs=None, library_dirs=None, macros=None):
+def test_compile(build_ext, name, code, libraries=None, include_dirs=None, library_dirs=None, macros=None,
+                 extra_preargs=None):
     test_compile_dir = os.path.join(build_ext.build_temp, 'test_compile')
     if not os.path.exists(test_compile_dir):
         os.makedirs(test_compile_dir)
@@ -170,7 +201,7 @@ def test_compile(build_ext, name, code, libraries=None, include_dirs=None, libra
     shared_object_file = compiler.shared_object_filename(
         name, output_dir=test_compile_dir)
 
-    compiler.compile([source_file], extra_preargs=['-std=c++11'],
+    compiler.compile([source_file], extra_preargs=extra_preargs,
                      include_dirs=include_dirs, macros=macros)
     compiler.link_shared_object(
         [object_file], shared_object_file, libraries=libraries, library_dirs=library_dirs)
@@ -178,7 +209,7 @@ def test_compile(build_ext, name, code, libraries=None, include_dirs=None, libra
     return shared_object_file
 
 
-def get_cuda_dirs(build_ext):
+def get_cuda_dirs(build_ext, cpp_flags):
     cuda_include_dirs = []
     cuda_lib_dirs = []
 
@@ -202,7 +233,7 @@ def get_cuda_dirs(build_ext):
 
     try:
         test_compile(build_ext, 'test_cuda', libraries=['cudart'], include_dirs=cuda_include_dirs,
-                     library_dirs=cuda_lib_dirs, code=textwrap.dedent('''\
+                     library_dirs=cuda_lib_dirs, extra_preargs=cpp_flags, code=textwrap.dedent('''\
             #include <cuda_runtime.h>
             void test() {
                 cudaSetDevice(0);
@@ -221,7 +252,7 @@ def get_cuda_dirs(build_ext):
     return cuda_include_dirs, cuda_lib_dirs
 
 
-def get_nccl_dirs(build_ext, cuda_include_dirs, cuda_lib_dirs):
+def get_nccl_dirs(build_ext, cuda_include_dirs, cuda_lib_dirs, cpp_flags):
     nccl_include_dirs = []
     nccl_lib_dirs = []
 
@@ -240,7 +271,7 @@ def get_nccl_dirs(build_ext, cuda_include_dirs, cuda_lib_dirs):
 
     try:
         test_compile(build_ext, 'test_nccl', libraries=['nccl'], include_dirs=nccl_include_dirs + cuda_include_dirs,
-                     library_dirs=nccl_lib_dirs + cuda_lib_dirs, code=textwrap.dedent('''\
+                     library_dirs=nccl_lib_dirs + cuda_lib_dirs, extra_preargs=cpp_flags, code=textwrap.dedent('''\
             #include <nccl.h>
             #if NCCL_MAJOR < 2
             #error Horovod requires NCCL 2.0 or later version, please upgrade.
@@ -266,7 +297,8 @@ def get_nccl_dirs(build_ext, cuda_include_dirs, cuda_lib_dirs):
 def fully_define_extension(build_ext):
     check_tf_version()
 
-    tf_compile_flags, tf_link_flags = get_tf_flags(build_ext)
+    cpp_flags = get_cpp_flags(build_ext)
+    tf_compile_flags, tf_link_flags = get_tf_flags(build_ext, cpp_flags)
     mpi_flags = get_mpi_flags()
 
     gpu_allreduce = os.environ.get('HOROVOD_GPU_ALLREDUCE')
@@ -286,7 +318,7 @@ def fully_define_extension(build_ext):
 
     if gpu_allreduce or gpu_allgather or gpu_broadcast:
         have_cuda = True
-        cuda_include_dirs, cuda_lib_dirs = get_cuda_dirs(build_ext)
+        cuda_include_dirs, cuda_lib_dirs = get_cuda_dirs(build_ext, cpp_flags)
     else:
         have_cuda = False
         cuda_include_dirs = cuda_lib_dirs = []
@@ -294,7 +326,7 @@ def fully_define_extension(build_ext):
     if gpu_allreduce == 'NCCL':
         have_nccl = True
         nccl_include_dirs, nccl_lib_dirs = get_nccl_dirs(
-            build_ext, cuda_include_dirs, cuda_lib_dirs)
+            build_ext, cuda_include_dirs, cuda_lib_dirs, cpp_flags)
     else:
         have_nccl = False
         nccl_include_dirs = nccl_lib_dirs = []
@@ -304,7 +336,7 @@ def fully_define_extension(build_ext):
     SOURCES = ['horovod/tensorflow/mpi_message.cc',
                'horovod/tensorflow/mpi_ops.cc',
                'horovod/tensorflow/timeline.cc']
-    COMPILE_FLAGS = ['-std=c++11', '-fPIC', '-O2'] + shlex.split(mpi_flags) + tf_compile_flags
+    COMPILE_FLAGS = cpp_flags + shlex.split(mpi_flags) + tf_compile_flags
     LINK_FLAGS = shlex.split(mpi_flags) + tf_link_flags
     LIBRARY_DIRS = []
     LIBRARIES = []


Test patch: