Repository: numba/numba

Issue:
CUDA parallel testing sometimes has an issue with the driver and/or runtime API
Lately I've occasionally been seeing something going wrong with initializing CUDA when running the testsuite in parallel:

```
$ python -m numba.runtests numba.cuda.tests.cudapy -vf -m
Parallel: 329. Serial: 585
test_signature_3 (numba.cuda.tests.cudapy.test_gufunc_scheduling.TestGUFuncScheduling) ... ok
test_signature_2 (numba.cuda.tests.cudapy.test_gufunc_scheduling.TestGUFuncScheduling) ... ok
test_signature_7 (numba.cuda.tests.cudapy.test_gufunc_scheduling.TestGUFuncScheduling) ... ok
test_signature_4 (numba.cuda.tests.cudapy.test_gufunc_scheduling.TestGUFuncScheduling) ... ok
test_signature_5 (numba.cuda.tests.cudapy.test_gufunc_scheduling.TestGUFuncScheduling) ... ok
test_signature_6 (numba.cuda.tests.cudapy.test_gufunc_scheduling.TestGUFuncScheduling) ... ok
test_signature_1 (numba.cuda.tests.cudapy.test_gufunc_scheduling.TestGUFuncScheduling) ... ok
test_const_string (numba.cuda.tests.cudapy.test_const_string.TestCudaConstString) ... ok
test_device_function (numba.cuda.tests.cudapy.test_compiler.TestCompileToPTX) ... ERROR

======================================================================
ERROR: test_device_function (numba.cuda.tests.cudapy.test_compiler.TestCompileToPTX)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/gmarkall/numbadev/numba/numba/cuda/tests/cudapy/test_compiler.py", line 33, in test_device_function
    ptx, resty = compile_ptx(add, args, device=True)
  File "/home/gmarkall/numbadev/numba/numba/core/compiler_lock.py", line 35, in _acquire_compile_lock
    return func(*args, **kwargs)
  File "/home/gmarkall/numbadev/numba/numba/cuda/compiler.py", line 112, in compile_ptx
    arch = nvvm.get_arch_option(*cc)
  File "/home/gmarkall/numbadev/numba/numba/cuda/cudadrv/nvvm.py", line 386, in get_arch_option
    arch = find_closest_arch((major, minor))
  File "/home/gmarkall/numbadev/numba/numba/cuda/cudadrv/nvvm.py", line 359, in find_closest_arch
    raise NvvmSupportError(msg)
numba.cuda.cudadrv.error.NvvmSupportError: No supported GPU compute capabilities found. Please check your cudatoolkit version matches your CUDA version.

----------------------------------------------------------------------
Ran 9 tests in 0.227s

FAILED (errors=1)
```

I see this occur with different branches.


Discussion:
Note - there's a chance that the cause is that I have a damaged CUDA installation, but I'm going to keep this issue open to track any further notes on this issue.
I just realised this always happens when I run the test package `numba.cuda.tests.cudapy` with the `-m` flag - something about the initialization of CUDA when `cudapy` runs first (as opposed to `cudadrv`) causes this.
`TestCompileToPTX` is erroring because it is marked as a normal test case (`unittest.TestCase`) and not a CUDA test case, on the assumption that it doesn't require a CUDA context. Some of its tests run in a child process that then tries to get the supported compute capabilities using `cudaRuntimeGetVersion` in order to use NVVM:

```
======================================================================
ERROR: test_device_function_with_debug (numba.cuda.tests.cudapy.test_compiler.TestCompileToPTX)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/gmarkall/numbadev/numba/numba/cuda/tests/cudapy/test_compiler.py", line 80, in test_device_function_with_debug
    ptx, resty = compile_ptx(f, [], device=True, debug=True)
  File "/home/gmarkall/numbadev/numba/numba/core/compiler_lock.py", line 35, in _acquire_compile_lock
    return func(*args, **kwargs)
  File "/home/gmarkall/numbadev/numba/numba/cuda/compiler.py", line 221, in compile_ptx
    ptx = lib.get_asm_str(cc=cc)
  File "/home/gmarkall/numbadev/numba/numba/cuda/codegen.py", line 110, in get_asm_str
    arch = nvvm.get_arch_option(*cc)
  File "/home/gmarkall/numbadev/numba/numba/cuda/cudadrv/nvvm.py", line 421, in get_arch_option
    arch = find_closest_arch((major, minor))
  File "/home/gmarkall/numbadev/numba/numba/cuda/cudadrv/nvvm.py", line 389, in find_closest_arch
    supported_cc = get_supported_ccs()
  File "/home/gmarkall/numbadev/numba/numba/cuda/cudadrv/nvvm.py", line 348, in get_supported_ccs
    cudart_version_major, cudart_version_minor = runtime.get_version()
  File "/home/gmarkall/numbadev/numba/numba/cuda/cudadrv/runtime.py", line 113, in get_version
    self.cudaRuntimeGetVersion(ctypes.byref(rtver))
  File "/home/gmarkall/numbadev/numba/numba/cuda/cudadrv/runtime.py", line 82, in safe_cuda_api_call
    self._check_error(fname, retcode)
  File "/home/gmarkall/numbadev/numba/numba/cuda/cudadrv/runtime.py", line 91, in _check_error
    raise CudaRuntimeAPIError(retcode, msg)
numba.cuda.cudadrv.runtime.CudaRuntimeAPIError: [3] Call to cudaRuntimeGetVersion results in CUDA_ERROR_NOT_INITIALIZED
```

For some child processes it returns that error. 

Resolving patch:
diff --git a/numba/cuda/cudadrv/nvvm.py b/numba/cuda/cudadrv/nvvm.py
--- a/numba/cuda/cudadrv/nvvm.py
+++ b/numba/cuda/cudadrv/nvvm.py
@@ -349,6 +349,7 @@ def get_supported_ccs():
     except: # noqa: E722
         # The CUDA Runtime may not be present
         cudart_version_major = 0
+        cudart_version_minor = 0
 
     ctk_ver = f"{cudart_version_major}.{cudart_version_minor}"
     unsupported_ver = f"CUDA Toolkit {ctk_ver} is unsupported by Numba - " \


Test patch:
diff --git a/numba/cuda/testing.py b/numba/cuda/testing.py
--- a/numba/cuda/testing.py
+++ b/numba/cuda/testing.py
@@ -37,6 +37,20 @@ def tearDown(self):
         reset()
 
 
+def ensure_supported_ccs_initialized():
+    from numba.cuda import is_available as cuda_is_available
+    from numba.cuda.cudadrv import nvvm
+
+    if cuda_is_available():
+        # Ensure that cudart.so is loaded and the list of supported compute
+        # capabilities in the nvvm module is populated before a fork. This is
+        # needed because some compilation tests don't require a CUDA context,
+        # but do use NVVM, and it is required that libcudart.so should be
+        # loaded before a fork (note that the requirement is not explicitly
+        # documented).
+        nvvm.get_supported_ccs()
+
+
 def skip_on_cudasim(reason):
     """Skip this test if running on the CUDA simulator"""
     return unittest.skipIf(config.ENABLE_CUDASIM, reason)
diff --git a/numba/cuda/tests/__init__.py b/numba/cuda/tests/__init__.py
--- a/numba/cuda/tests/__init__.py
+++ b/numba/cuda/tests/__init__.py
@@ -1,3 +1,4 @@
+from numba.cuda.testing import ensure_supported_ccs_initialized
 from numba.testing import unittest
 from numba.testing import load_testsuite
 from numba import cuda
@@ -7,15 +8,9 @@
 def load_tests(loader, tests, pattern):
     suite = unittest.TestSuite()
     this_dir = dirname(__file__)
+    ensure_supported_ccs_initialized()
     suite.addTests(load_testsuite(loader, join(this_dir, 'nocuda')))
     if cuda.is_available():
-        # Ensure that cudart.so is loaded and the list of supported compute
-        # capabilities in the nvvm module is populated before a fork. This is
-        # needed because some compilation tests don't require a CUDA context,
-        # but do use NVVM to compile, and it is required that libcudart.so
-        # should be loaded before a fork (note that the requirement is not
-        # explicitly documented).
-        cuda.cudadrv.nvvm.get_supported_ccs()
         suite.addTests(load_testsuite(loader, join(this_dir, 'cudasim')))
         gpus = cuda.list_devices()
         if gpus and gpus[0].compute_capability >= (2, 0):
diff --git a/numba/cuda/tests/cudadrv/__init__.py b/numba/cuda/tests/cudadrv/__init__.py
--- a/numba/cuda/tests/cudadrv/__init__.py
+++ b/numba/cuda/tests/cudadrv/__init__.py
@@ -1,6 +1,8 @@
+from numba.cuda.testing import ensure_supported_ccs_initialized
 from numba.testing import load_testsuite
 import os
 
 
 def load_tests(loader, tests, pattern):
+    ensure_supported_ccs_initialized()
     return load_testsuite(loader, os.path.dirname(__file__))
diff --git a/numba/cuda/tests/cudapy/__init__.py b/numba/cuda/tests/cudapy/__init__.py
--- a/numba/cuda/tests/cudapy/__init__.py
+++ b/numba/cuda/tests/cudapy/__init__.py
@@ -1,6 +1,8 @@
+from numba.cuda.testing import ensure_supported_ccs_initialized
 from numba.testing import load_testsuite
 import os
 
 
 def load_tests(loader, tests, pattern):
+    ensure_supported_ccs_initialized()
     return load_testsuite(loader, os.path.dirname(__file__))
diff --git a/numba/cuda/tests/nocuda/__init__.py b/numba/cuda/tests/nocuda/__init__.py
--- a/numba/cuda/tests/nocuda/__init__.py
+++ b/numba/cuda/tests/nocuda/__init__.py
@@ -1,6 +1,8 @@
+from numba.cuda.testing import ensure_supported_ccs_initialized
 from numba.testing import load_testsuite
 import os
 
 
 def load_tests(loader, tests, pattern):
+    ensure_supported_ccs_initialized()
     return load_testsuite(loader, os.path.dirname(__file__))
diff --git a/numba/cuda/tests/nocuda/test_nvvm.py b/numba/cuda/tests/nocuda/test_nvvm.py
--- a/numba/cuda/tests/nocuda/test_nvvm.py
+++ b/numba/cuda/tests/nocuda/test_nvvm.py
@@ -1,7 +1,6 @@
-from numba.cuda.compiler import compile_kernel
 from numba.cuda.cudadrv import nvvm
-from numba.cuda.testing import skip_on_cudasim, SerialMixin
-from numba.core import types, utils
+from numba.cuda.testing import skip_on_cudasim
+from numba.core import utils
 
 from llvmlite import ir
 from llvmlite import binding as llvm
@@ -19,21 +18,7 @@
 @skip_on_cudasim('libNVVM not supported in simulator')
 @unittest.skipIf(utils.MACHINE_BITS == 32, "CUDA not support for 32-bit")
 @unittest.skipIf(not nvvm.is_available(), "No libNVVM")
-class TestNvvmWithoutCuda(SerialMixin, unittest.TestCase):
-    def test_nvvm_llvm_to_ptx(self):
-        """
-        A simple test to exercise nvvm.llvm_to_ptx()
-        to trigger issues with mismatch NVVM API.
-        """
-
-        def foo(x):
-            x[0] = 123
-
-        cukern = compile_kernel(foo, args=(types.int32[::1],), link=())
-        llvmir = cukern._codelibrary.get_llvm_str()
-        ptx = nvvm.llvm_to_ptx(llvmir)
-        self.assertIn("foo", ptx.decode('ascii'))
-
+class TestNvvmWithoutCuda(unittest.TestCase):
     def test_nvvm_memset_fixup_for_34(self):
         """
         Test llvm.memset changes in llvm7.