Repository: cupy/cupy

Issue:
Re-enable `CUPY_NUM_BUILD_JOBS` option
I'm currently investigating the issue reported in #4339.

## Without parallelism

https://jenkins.preferred.jp/job/chainer/job/other_cupy_release_builder/133/CUDA=11.1,PYTHON=3.7.0,label=mn1-p100/consoleFull

```
(1) Compile (thrust.cpp -> thrust.o)
09:05:17 gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -D_FORCE_INLINES=1 -DCUPY_CUB_VERSION_CODE=100910 -DCUPY_JITIFY_VERSION_CODE=-1 -I/usr/local/cuda/include/cub -I/work/cupy/install/../cupy/core/include -I/usr/local/cuda/include -I/opt/pyenv/versions/3.7.0/include/python3.7m -c cupy/cuda/thrust.cpp -o build/temp.linux-x86_64-3.7/cupy/cuda/thrust.o

(2) Compile (cupy_thrust.cu -> cupy_thrust.o)
09:05:20 /usr/local/cuda/bin/nvcc -D_FORCE_INLINES=1 -DCUPY_CUB_VERSION_CODE=100910 -DCUPY_JITIFY_VERSION_CODE=-1 -I/usr/local/cuda/include/cub -I/work/cupy/install/../cupy/core/include -I/usr/local/cuda/include -I/opt/pyenv/versions/3.7.0/include/python3.7m -c cupy/cuda/cupy_thrust.cu -o build/temp.linux-x86_64-3.7/cupy/cuda/cupy_thrust.o --generate-code=arch=compute_35,code=compute_35 --generate-code=arch=compute_50,code=compute_50 --generate-code=arch=compute_60,code=sm_60 --generate-code=arch=compute_61,code=sm_61 --generate-code=arch=compute_70,code=sm_70 --generate-code=arch=compute_75,code=sm_75 --generate-code=arch=compute_80,code=sm_80 --generate-code=arch=compute_80,code=compute_80 -O2 --compiler-options="-fPIC" --std=c++11

(3) Link (thrust.*.so)
09:24:46 g++ -pthread -shared -L/opt/pyenv/versions/3.7.0/lib -L/opt/pyenv/versions/3.7.0/lib build/temp.linux-x86_64-3.7/cupy/cuda/thrust.o build/temp.linux-x86_64-3.7/cupy/cuda/cupy_thrust.o -L/usr/local/cuda/lib64 -lcudart -o build/lib.linux-x86_64-3.7/cupy/cuda/thrust.cpython-37m-x86_64-linux-gnu.so -Wl,--disable-new-dtags,-rpath,$ORIGIN/../../cupy/.data/lib
```

## With parallelism

https://jenkins.preferred.jp/job/chainer/job/other_cupy_release_builder/132/CUDA=11.1,PYTHON=3.7.0,label=mn1-p100/consoleFull

```
(1) Compile (thrust.cpp -> thrust.o)  -> nvcc used here (gcc expected)
02:44:26 /usr/local/cuda/bin/nvcc -D_FORCE_INLINES=1 -DCUPY_CUB_VERSION_CODE=100910 -DCUPY_JITIFY_VERSION_CODE=-1 -I/usr/local/cuda/include/cub -I/work/cupy/install/../cupy/core/include -I/usr/local/cuda/include -I/opt/pyenv/versions/3.7.0/include/python3.7m -c cupy/cuda/thrust.cpp -o build/temp.linux-x86_64-3.7/cupy/cuda/thrust.o

(2) Compile (cupy_thrust.cu -> cupy_thrust.o)
02:44:27 /usr/local/cuda/bin/nvcc -D_FORCE_INLINES=1 -DCUPY_CUB_VERSION_CODE=100910 -DCUPY_JITIFY_VERSION_CODE=-1 -I/usr/local/cuda/include/cub -I/work/cupy/install/../cupy/core/include -I/usr/local/cuda/include -I/opt/pyenv/versions/3.7.0/include/python3.7m -c cupy/cuda/cupy_thrust.cu -o build/temp.linux-x86_64-3.7/cupy/cuda/cupy_thrust.o --generate-code=arch=compute_35,code=compute_35 --generate-code=arch=compute_50,code=compute_50 --generate-code=arch=compute_60,code=sm_60 --generate-code=arch=compute_61,code=sm_61 --generate-code=arch=compute_70,code=sm_70 --generate-code=arch=compute_75,code=sm_75 --generate-code=arch=compute_80,code=sm_80 --generate-code=arch=compute_80,code=compute_80 -O2 --compiler-options="-fPIC" --std=c++11

(3) Link (thrust.*.so)  -> Error
03:00:25 g++ -pthread -shared -L/opt/pyenv/versions/3.7.0/lib -L/opt/pyenv/versions/3.7.0/lib build/temp.linux-x86_64-3.7/cupy/cuda/thrust.o build/temp.linux-x86_64-3.7/cupy/cuda/cupy_thrust.o -L/usr/local/cuda/lib64 -lcudart -o build/lib.linux-x86_64-3.7/cupy/cuda/thrust.cpython-37m-x86_64-linux-gnu.so -Wl,--disable-new-dtags,-rpath,$ORIGIN/../../cupy/.data/lib
03:00:26 /opt/rh/devtoolset-7/root/usr/libexec/gcc/x86_64-redhat-linux/7/ld: build/temp.linux-x86_64-3.7/cupy/cuda/thrust.o: relocation R_X86_64_32S against `.rodata' can not be used when making a shared object; recompile with -fPIC
03:00:26 /opt/rh/devtoolset-7/root/usr/libexec/gcc/x86_64-redhat-linux/7/ld: final link failed: Nonrepresentable section on output
```


Discussion:

Resolving patch:
diff --git a/cupy_setup_build.py b/cupy_setup_build.py
--- a/cupy_setup_build.py
+++ b/cupy_setup_build.py
@@ -928,47 +928,42 @@ def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts):
                 self, obj, src, ext, cc_args, extra_postargs, pp_opts)
 
         if use_hip:
-            return self._comiple_unix_hipcc(
+            return self._compile_unix_hipcc(
                 obj, src, ext, cc_args, extra_postargs, pp_opts)
         else:
-            return self._comiple_unix_nvcc(
+            return self._compile_unix_nvcc(
                 obj, src, ext, cc_args, extra_postargs, pp_opts)
 
-    def _comiple_unix_nvcc(self,
+    def _compile_unix_nvcc(self,
                            obj, src, ext, cc_args, extra_postargs, pp_opts):
         # For CUDA C source files, compile them with NVCC.
-        _compiler_so = self.compiler_so
-        try:
-            nvcc_path = build.get_nvcc_path()
-            base_opts = build.get_compiler_base_options()
-            self.set_executable('compiler_so', nvcc_path)
-
-            cuda_version = build.get_cuda_version()
-            postargs = _nvcc_gencode_options(cuda_version) + [
-                '-O2', '--compiler-options="-fPIC"', '--std=c++11']
-            print('NVCC options:', postargs)
+        nvcc_path = build.get_nvcc_path()
+        base_opts = build.get_compiler_base_options()
+        compiler_so = nvcc_path
 
-            return unixccompiler.UnixCCompiler._compile(
-                self, obj, src, ext, base_opts + cc_args, postargs, pp_opts)
-        finally:
-            self.compiler_so = _compiler_so
+        cuda_version = build.get_cuda_version()
+        postargs = _nvcc_gencode_options(cuda_version) + [
+            '-O2', '--compiler-options="-fPIC"', '--std=c++11']
+        print('NVCC options:', postargs)
+        try:
+            self.spawn(compiler_so + base_opts + cc_args + [src, '-o', obj] +
+                       postargs)
+        except errors.DistutilsExecError as e:
+            raise errors.CompileError(str(e))
 
-    def _comiple_unix_hipcc(self,
+    def _compile_unix_hipcc(self,
                             obj, src, ext, cc_args, extra_postargs, pp_opts):
         # For CUDA C source files, compile them with HIPCC.
-        _compiler_so = self.compiler_so
+        rocm_path = build.get_hipcc_path()
+        base_opts = build.get_compiler_base_options()
+        compiler_so = rocm_path
+        postargs = ['-O2', '-fPIC', '--include', 'hip_runtime.h']
+        print('HIPCC options:', postargs)
         try:
-            rocm_path = build.get_hipcc_path()
-            base_opts = build.get_compiler_base_options()
-            self.set_executable('compiler_so', rocm_path)
-
-            postargs = ['-O2', '-fPIC', '--include', 'hip_runtime.h']
-            print('HIPCC options:', postargs)
-
-            return unixccompiler.UnixCCompiler._compile(
-                self, obj, src, ext, base_opts + cc_args, postargs, pp_opts)
-        finally:
-            self.compiler_so = _compiler_so
+            self.spawn(compiler_so + base_opts + cc_args + [src, '-o', obj] +
+                       postargs)
+        except errors.DistutilsExecError as e:
+            raise errors.CompileError(str(e))
 
     def link(self, target_desc, objects, output_filename, *args):
         use_hipcc = False
@@ -1087,8 +1082,7 @@ def _wrap_new_compiler(*args, **kwargs):
         build_ext.build_ext.run(self)
 
     def build_extensions(self):
-        # TODO(kmaehashi): This option may be unstable, under investigation
-        num_jobs = int(os.environ.get('CUPY_NUM_BUILD_JOBS', '1'))
+        num_jobs = int(os.environ.get('CUPY_NUM_BUILD_JOBS', '4'))
         if num_jobs > 1:
             self.parallel = num_jobs
         super().build_extensions()


Test patch: