Repository: pytorch/pytorch

Issue:
Strange behavior when -fsanitize=address added to CMAKE_C_FLAGS/CMAKE_CXX_FLAGS
Apply:

```
diff --git a/aten/CMakeLists.txt b/aten/CMakeLists.txt
index 9893cc4..7a1997a 100644
--- a/aten/CMakeLists.txt
+++ b/aten/CMakeLists.txt
@@ -26,8 +26,8 @@ if("${isSystemDir}" STREQUAL "-1")
 endif()
 
 IF(NOT MSVC)
-  set(CMAKE_CXX_FLAGS "--std=c++11 -Wall -Wno-unknown-pragmas -Wno-vla -fexceptions ${CMAKE_CXX_FLAGS}")
-  set(CMAKE_C_FLAGS "-fexceptions ${CMAKE_C_FLAGS}")
+  set(CMAKE_CXX_FLAGS "-fsanitize=address --std=c++11 -Wall -Wno-unknown-pragmas -Wno-vla -fexceptions ${CMAKE_CXX_FLAGS}")
+  set(CMAKE_C_FLAGS "-fsanitize=address -fexceptions ${CMAKE_C_FLAGS}")
 ENDIF(NOT MSVC)
 
 
```

and then try to build. It'll fail, with the log tail looking like

```
-- Performing Test CXX_HAS_SSE4_2_3 - Failed
-- Performing Test CXX_HAS_SSE4_2_4
-- Performing Test CXX_HAS_SSE4_2_4 - Failed
-- Performing Test CXX_HAS_AVX_1
-- Performing Test CXX_HAS_AVX_1 - Failed
-- Performing Test CXX_HAS_AVX_2
-- Performing Test CXX_HAS_AVX_2 - Failed
-- Performing Test CXX_HAS_AVX_3
-- Performing Test CXX_HAS_AVX_3 - Failed
-- Performing Test CXX_HAS_AVX2_1
-- Performing Test CXX_HAS_AVX2_1 - Failed
-- Performing Test CXX_HAS_AVX2_2
-- Performing Test CXX_HAS_AVX2_2 - Failed
-- Performing Test CXX_HAS_AVX2_3
-- Performing Test CXX_HAS_AVX2_3 - Failed
-- Performing Test HAS_C11_ATOMICS
-- Performing Test HAS_C11_ATOMICS - Failed
-- Performing Test HAS_MSC_ATOMICS
-- Performing Test HAS_MSC_ATOMICS - Failed
-- Performing Test HAS_GCC_ATOMICS
-- Performing Test HAS_GCC_ATOMICS - Failed
-- Looking for pthread.h
-- Looking for pthread.h - found
-- Looking for pthread_create
-- Looking for pthread_create - found
-- Found Threads: TRUE  
CMake Error at CMakeLists.txt:313 (TARGET_LINK_LIBRARIES):
  Cannot specify link libraries for target "TH" which is not built by this
  project.


-- Configuring incomplete, errors occurred!
```

In contrast, on this machine, I'd expect some of the feature tests to pass. Here is log snippet from a successful run without the patch:

```
-- Performing Test CXX_HAS_SSE4_2_2
-- Performing Test CXX_HAS_SSE4_2_2 - Success
-- Performing Test CXX_HAS_AVX_1
-- Performing Test CXX_HAS_AVX_1 - Failed
-- Performing Test CXX_HAS_AVX_2
-- Performing Test CXX_HAS_AVX_2 - Success
-- Performing Test CXX_HAS_AVX2_1
-- Performing Test CXX_HAS_AVX2_1 - Failed
-- Performing Test CXX_HAS_AVX2_2
-- Performing Test CXX_HAS_AVX2_2 - Success
-- SSE2 Found
-- SSE3 Found
-- AVX Found
-- AVX2 Found
-- Performing Test HAS_C11_ATOMICS
-- Performing Test HAS_C11_ATOMICS - Failed
-- Performing Test HAS_MSC_ATOMICS
-- Performing Test HAS_MSC_ATOMICS - Failed
-- Performing Test HAS_GCC_ATOMICS
-- Performing Test HAS_GCC_ATOMICS - Success
```

An example error log entry:

```
Performing C SOURCE FILE Test HAS_GCC_ATOMICS failed with the following output:
Change Dir: /home/ezyang/Dev/pytorch-tmp/torch/lib/build/aten/CMakeFiles/CMakeTmp

Run Build Command:"/bin/gmake" "cmTC_67e0e/fast"
/bin/gmake -f CMakeFiles/cmTC_67e0e.dir/build.make CMakeFiles/cmTC_67e0e.dir/build
gmake[1]: Entering directory `/data/users/ezyang/pytorch-tmp/torch/lib/build/aten/CMakeFiles/CMakeTmp'
Building C object CMakeFiles/cmTC_67e0e.dir/src.c.o
/home/ezyang/Dev/pytorch-tmp-env/bin/cc    -DCUDA_HAS_FP16=1 -fsanitize=address -fexceptions  -fopenmp -DHAVE_GCC_GET_CPUID -DHAS_GCC_ATOMICS -fPIE   -o CMakeFiles/cm
TC_67e0e.dir/src.c.o   -c /home/ezyang/Dev/pytorch-tmp/torch/lib/build/aten/CMakeFiles/CMakeTmp/src.c
Linking C executable cmTC_67e0e
/usr/bin/cmake -E cmake_link_script CMakeFiles/cmTC_67e0e.dir/link.txt --verbose=1
/home/ezyang/Dev/pytorch-tmp-env/bin/cc  -DCUDA_HAS_FP16=1 -fsanitize=address -fexceptions  -fopenmp -DHAVE_GCC_GET_CPUID -DHAS_GCC_ATOMICS    CMakeFiles/cmTC_67e0e.dir/src.c.o  -o cmTC_67e0e -rdynamic 
gmake[1]: Leaving directory `/data/users/ezyang/pytorch-tmp/torch/lib/build/aten/CMakeFiles/CMakeTmp'

Return value: 127
Source file was:

int main()
{
  int a;
  __sync_lock_test_and_set(&a, 1);
  __sync_fetch_and_add(&a, 1);
  if(!__sync_bool_compare_and_swap(&a, 2, 3))
    return -1;
  return 0;
}
```

No further stderr, which is pretty weird.

It would be nice to get a better cmake error message in this situation.

Google juice: ASAN.


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -1,3 +1,83 @@
+# Welcome to the PyTorch setup.py.
+#
+# Environment variables you are probably interestd in:
+#
+#   DEBUG
+#     build with -O0 and -g (debug symbols)
+#
+#   MAX_JOBS
+#     maximum number of compile jobs we should use to compile your code
+#
+#   NO_CUDA
+#     disables CUDA build
+#
+#   CFLAGS
+#     flags to apply to both C and C++ files to be compiled (a quirk of setup.py
+#     which we have faithfully adhered to in our build system is that CFLAGS
+#     also applies to C++ files, in contrast to the default behavior of autogoo
+#     and cmake build systems.)
+#
+#   CC
+#     the C/C++ compiler to use (NB: the CXX flag has no effect for distutils
+#     compiles, because distutils always uses CC to compile, even for C++
+#     files.
+#
+# Environment variables for feature toggles:
+#
+#   NO_CUDNN
+#     disables the cuDNN build
+#
+#   NO_NNPACK
+#     disables NNPACK build
+#
+#   NO_DISTRIBUTED
+#     disables THD (distributed) build
+#
+#   NO_SYSTEM_NCCL
+#     disables use of system-wide nccl (we will use our submoduled
+#     copy in torch/lib/nccl)
+#
+#   WITH_GLOO_IBVERBS
+#     toggle features related to distributed support
+#
+#   WITH_SCALARS
+#     build with native support for scalars (zero-dim tensors)
+#
+#   PYTORCH_BINARY_BUILD
+#     toggle static linking against libstdc++, used when we're building
+#     binaries for distribution
+#
+#   PYTORCH_BUILD_VERSION
+#   PYTORCH_BUILD_NUMBER
+#     specify the version of PyTorch, rather than the hard-coded version
+#     in this file; used when we're building binaries for distribution
+#
+# Environment variables we respect (these environment variables are
+# conventional and are often understood/set by other software.)
+#
+#   CUDA_HOME (Linux/OS X)
+#   CUDA_PATH (Windows)
+#     specify where CUDA is installed; usually /usr/local/cuda or
+#     /usr/local/cuda-x.y
+#
+#   CUDNN_LIB_DIR
+#   CUDNN_INCLUDE_DIR
+#   CUDNN_LIBRARY
+#     specify where cuDNN is installed
+#
+#   NCCL_ROOT_DIR
+#   NCCL_LIB_DIR
+#   NCCL_INCLUDE_DIR
+#     specify where nccl is installed
+#
+#   NVTOOLSEXT_PATH (Windows only)
+#     specify where nvtoolsext is installed
+#
+#   LIBRARY_PATH
+#   LD_LIBRARY_PATH
+#     we will search for libraries in these paths
+
+
 from setuptools import setup, Extension, distutils, Command, find_packages
 import setuptools.command.build_ext
 import setuptools.command.install
@@ -9,6 +89,7 @@
 import platform
 import subprocess
 import shutil
+import multiprocessing
 import sys
 import os
 import json
@@ -38,6 +119,11 @@
     os.environ['WITH_SCALARS'] = '1'
 WITH_SCALARS = check_env_flag('WITH_SCALARS')
 
+NUM_JOBS = multiprocessing.cpu_count()
+max_jobs = os.getenv("MAX_JOBS")
+if max_jobs is not None:
+    NUM_JOBS = min(NUM_JOBS, int(max_jobs))
+
 try:
     import ninja
     WITH_NINJA = True
@@ -63,11 +149,7 @@ def parallelCCompile(self, sources, output_dir=None, macros=None,
         def _single_compile(obj):
             src, ext = build[obj]
             self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
-        num_jobs = multiprocessing.cpu_count()
-        max_jobs = os.getenv("MAX_JOBS")
-        if max_jobs is not None:
-            num_jobs = min(num_jobs, int(max_jobs))
-        multiprocessing.pool.ThreadPool(num_jobs).map(_single_compile, objects)
+        multiprocessing.pool.ThreadPool(NUM_JOBS).map(_single_compile, objects)
 
         return objects
     distutils.ccompiler.CCompiler.compile = parallelCCompile
@@ -120,6 +202,7 @@ def build_libs(libs):
         build_libs_cmd = ['bash', 'torch/lib/build_libs.sh']
     my_env = os.environ.copy()
     my_env["PYTORCH_PYTHON"] = sys.executable
+    my_env["NUM_JOBS"] = str(NUM_JOBS)
     if not IS_WINDOWS:
         if WITH_NINJA:
             my_env["CMAKE_GENERATOR"] = '-GNinja'


Test patch: