Repository: cupy/cupy

Issue:
cupy.cuda.compiler.CompileException: nvrtc: error
Hello,
I met a problem about cuda when I running the code.
I had correctly installed cuda but it does not work.
when I run,
>>>python3 pose_detector1.py posenet models/coco_posenet.npz --img data/person.png -g 0

The error shows:

Loading PoseNet...
Inference scale: 1.5...
NVRTC compilation error: nvrtc: error: invalid value for --gpu-architecture (-arch)

-----
Name: 
Options: -I/usr/local/lib/python3.5/dist-packages/cupy-4.0.0b2-py3.5-linux-x86_64.egg/cupy/core/include -ftz=true -arch=compute_61
CUDA source:
1 
-----
Traceback (most recent call last):
  File "/usr/local/lib/python3.5/dist-packages/cupy-4.0.0b2-py3.5-linux-x86_64.egg/cupy/cuda/compiler.py", line 229, in compile
    nvrtc.compileProgram(self.ptr, options)
  File "cupy/cuda/nvrtc.pyx", line 98, in cupy.cuda.nvrtc.compileProgram
  File "cupy/cuda/nvrtc.pyx", line 108, in cupy.cuda.nvrtc.compileProgram
  File "cupy/cuda/nvrtc.pyx", line 53, in cupy.cuda.nvrtc.check_status
cupy.cuda.nvrtc.NVRTCError: NVRTC_ERROR_INVALID_OPTION (5)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "pose_detector1.py", line 509, in <module>
    person_pose_array = pose_detector(img)
  File "pose_detector1.py", line 435, in __call__
    h1s, h2s = self.model(x_data)
  File "/home/suiishii/Chainer_Realtime_Multi-Person_Pose_Estimation/models/CocoPoseNet.py", line 136, in __call__
    h = F.relu(self.conv1_1(x))
  File "/usr/local/lib/python3.5/dist-packages/chainer/links/connection/convolution_2d.py", line 156, in __call__
    x, self.W, self.b, self.stride, self.pad)
  File "/usr/local/lib/python3.5/dist-packages/chainer/functions/connection/convolution_2d.py", line 467, in convolution_2d
    y, = fnode.apply(args)
  File "/usr/local/lib/python3.5/dist-packages/chainer/function_node.py", line 245, in apply
    outputs = self.forward(in_data)
  File "/usr/local/lib/python3.5/dist-packages/chainer/function_node.py", line 337, in forward
    return self.forward_gpu(inputs)
  File "/usr/local/lib/python3.5/dist-packages/chainer/functions/connection/convolution_2d.py", line 153, in forward_gpu
    x = cuda.cupy.ascontiguousarray(x)
  File "/usr/local/lib/python3.5/dist-packages/cupy-4.0.0b2-py3.5-linux-x86_64.egg/cupy/creation/from_data.py", line 92, in ascontiguousarray
    return core.ascontiguousarray(a, dtype)
  File "cupy/core/core.pyx", line 2131, in cupy.core.core.ascontiguousarray
  File "cupy/core/core.pyx", line 2142, in cupy.core.core.ascontiguousarray
  File "cupy/core/elementwise.pxi", line 823, in cupy.core.core.ufunc.__call__
  File "cupy/util.pyx", line 39, in cupy.util.memoize.decorator.ret
  File "cupy/core/elementwise.pxi", line 622, in cupy.core.core._get_ufunc_kernel
  File "cupy/core/elementwise.pxi", line 33, in cupy.core.core._get_simple_elementwise_kernel
  File "cupy/core/carray.pxi", line 170, in cupy.core.core.compile_with_cache
  File "/usr/local/lib/python3.5/dist-packages/cupy-4.0.0b2-py3.5-linux-x86_64.egg/cupy/cuda/compiler.py", line 123, in compile_with_cache
    base = _preprocess('', options, arch)
  File "/usr/local/lib/python3.5/dist-packages/cupy-4.0.0b2-py3.5-linux-x86_64.egg/cupy/cuda/compiler.py", line 86, in _preprocess
    result = prog.compile(options)
  File "/usr/local/lib/python3.5/dist-packages/cupy-4.0.0b2-py3.5-linux-x86_64.egg/cupy/cuda/compiler.py", line 233, in compile
    raise CompileException(log, self.src, self.name, options)
cupy.cuda.compiler.CompileException: nvrtc: error: invalid value for --gpu-architecture (-arch)

Here shows the information of cuda:
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 387.34                 Driver Version: 387.34                    |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  GeForce GTX 1080    Off  | 00000000:01:00.0  On |                  N/A |
| 33%   44C    P8    11W / 200W |   1313MiB /  8114MiB |      1%      Default |
+-------------------------------+----------------------+----------------------+
                                                                               
+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID   Type   Process name                             Usage      |
|=============================================================================|
|    0      1260      G   /usr/lib/xorg/Xorg                            47MiB |
|    0      2922      G   /usr/lib/xorg/Xorg                           295MiB |
|    0      3552      G   compiz                                       140MiB |
|    0      3916      G   fcitx-qimpanel                                 8MiB |
|    0     13646      G   ...-token=6178575BADBB10006C9A8F872A29596F   159MiB |
|    0     22760      C   python3                                      109MiB |
|    0     22903      C   python3                                      109MiB |
|    0     23080      C   python3                                      109MiB |
|    0     27219      C   python3                                      109MiB |
|    0     27329      C   python3                                      109MiB |
+-----------------------------------------------------------------------------+

I checked all the information of cuda, but I did not find some problem
Please help me to solve this issue.
Thank you！



Discussion:
@suishii Thank you for reporting a issue. Are you using CUDA 8.0?

@okuta I want to confirm this case with you.

* Compute capability of GeForce GTX 1080 is `61`. So we're passing `compute_61` to NVRTC.  https://github.com/cupy/cupy/blob/v4.0.0rc1/cupy/cuda/compiler.py#L26-L28
* However, NVRTC in CUDA 8.0 does not support `-arch=compute_61` (although `nvcc` supports it). The maximum value allowed is `compute_53` in NVRTC.
  Maybe we should handle this special case?
I see.
It is better to use latest compute capability number of CUDA in CuPy.


OK, I’ll fix that way.

Resolving patch:
diff --git a/cupy/cuda/compiler.py b/cupy/cuda/compiler.py
--- a/cupy/cuda/compiler.py
+++ b/cupy/cuda/compiler.py
@@ -13,6 +13,7 @@
 from cupy.cuda import nvrtc
 
 _nvrtc_version = None
+_nvrtc_max_compute_capability = None
 
 
 def _get_nvrtc_version():
@@ -24,7 +25,18 @@ def _get_nvrtc_version():
 
 
 def _get_arch():
-    cc = device.Device().compute_capability
+    global _nvrtc_max_compute_capability
+    if _nvrtc_max_compute_capability is None:
+        # See Supported Compile Options section of NVRTC User Guide for
+        # the maximum value allowed for `--gpu-architecture`.
+        major, minor = _get_nvrtc_version()
+        if major < 9:
+            # CUDA 7.0 / 7.5 / 8.0
+            _nvrtc_max_compute_capability = '53'
+        else:
+            # CUDA 9.0 / 9.1
+            _nvrtc_max_compute_capability = '72'
+    cc = min(device.Device().compute_capability, _nvrtc_max_compute_capability)
     return 'compute_%s' % cc
 
 


Test patch:
diff --git a/tests/cupy_tests/cuda_tests/test_compiler.py b/tests/cupy_tests/cuda_tests/test_compiler.py
--- a/tests/cupy_tests/cuda_tests/test_compiler.py
+++ b/tests/cupy_tests/cuda_tests/test_compiler.py
@@ -1,11 +1,69 @@
 import unittest
 
+import mock
 import six
 
+import cupy
 from cupy.cuda import compiler
 from cupy import testing
 
 
+def cuda_version():
+    return cupy.cuda.runtime.runtimeGetVersion()
+
+
+@testing.gpu
+class TestNvrtcArch(unittest.TestCase):
+
+    def _check_get_arch(self, device_cc, expected_arch):
+        with mock.patch('cupy.cuda.device.Device') as device_class:
+            device_class.return_value.compute_capability = device_cc
+            assert compiler._get_arch() == expected_arch
+
+    @unittest.skipUnless(cuda_version() < 9000, 'Requires CUDA 8.x or earlier')
+    def test_get_arch_cuda8(self):
+        self._check_get_arch('52', 'compute_52')
+        self._check_get_arch('53', 'compute_53')
+        self._check_get_arch('54', 'compute_53')
+
+    @unittest.skipUnless(9000 <= cuda_version(), 'Requires CUDA 9.x or later')
+    def test_get_arch_cuda9(self):
+        self._check_get_arch('71', 'compute_71')
+        self._check_get_arch('72', 'compute_72')
+        self._check_get_arch('73', 'compute_72')
+
+    def _compile(self, arch):
+        compiler.compile_using_nvrtc('', arch=arch)
+
+    @unittest.skipUnless(cuda_version() < 9000, 'Requires CUDA 8.x or earlier')
+    def test_compile_cuda8(self):
+        # This test is intended to detect specification change in NVRTC API.
+
+        # It should not fail.
+        # (Do not test `compute_53` as it is for Tegra.)
+        self._compile('compute_52')
+
+        # It should fail.
+        # (`compute_60` and `compute_61` are not supported by NVRTC in CUDA 8
+        #  but it does not raise error when used.)
+        self.assertRaises(
+            compiler.CompileException, self._compile, 'compute_54')
+        self.assertRaises(
+            compiler.CompileException, self._compile, 'compute_70')
+
+    @unittest.skipUnless(9000 <= cuda_version(), 'Requires CUDA 9.0 or later')
+    def test_compile_cuda9(self):
+        # This test is intended to detect specification change in NVRTC API.
+
+        # It should not fail.
+        # (Do not test `compute_72` as it is for Tegra.)
+        self._compile('compute_70')
+
+        # It should fail.
+        self.assertRaises(
+            compiler.CompileException, self._compile, 'compute_73')
+
+
 @testing.gpu
 class TestNvrtcStderr(unittest.TestCase):