Repository: mars-project/mars

Issue:
[BUG] Error will raise when CUDA driver installed but no devices detected
When ``libnvidia-ml.so`` can be detected while no devices is installed, an error will appear:

```
/usr/local/lib/python3.6/dist-packages/mars/lib/nvutils.py in get_driver_info()
    200     cuda_version = c_uint()
    201 
--> 202     _nvml_check_error(_nvml_lib.nvmlSystemGetDriverVersion(version_buf, len(version_buf)))
    203     _nvml_check_error(_nvml_lib.nvmlSystemGetCudaDriverVersion(byref(cuda_version)))
    204 

/usr/local/lib/python3.6/dist-packages/mars/lib/nvutils.py in _nvml_check_error(result)
    104         _error_str = _nvmlErrorString(result)
    105         if _error_str:
--> 106             raise NVError('NVML API Error %d: %s' % (result, _error_str.decode()))
    107         else:
    108             raise NVError('Unknown NVML API Error %d' % result)

NVError: NVML API Error 1: Uninitialized
```


Discussion:

Resolving patch:
diff --git a/mars/lib/nvutils.py b/mars/lib/nvutils.py
--- a/mars/lib/nvutils.py
+++ b/mars/lib/nvutils.py
@@ -31,10 +31,14 @@
 CU_DEVICE_ATTRIBUTE_PCI_DEVICE_ID = 34
 CU_DEVICE_ATTRIBUTE_MEMORY_CLOCK_RATE = 36
 
+CU_NO_CUDA_CAPABLE_DEVICE_DETECTED = 100
+
 # nvml constants
 NVML_SUCCESS = 0
 NVML_TEMPERATURE_GPU = 0
 
+NVML_DRIVER_NOT_LOADED = 9
+
 
 class _CUuuid_t(Structure):
     _fields_ = [
@@ -73,14 +77,36 @@ def _load_nv_library(*libnames):
 _nvml_device_status = namedtuple(
     '_nvml_device_status', 'gpu_util mem_util temperature fb_total_mem fb_used_mem fb_free_mem')
 
-
 _init_pid = None
 _gpu_count = None
 _driver_info = None
 _device_infos = dict()
 
+_no_device_warned = False
+
 
 class NVError(Exception):
+    def __init__(self, msg, *args, errno=None):
+        self._errno = errno
+        super().__init__(msg or 'Unknown error', *args)
+
+    def __str__(self):
+        return '(%s) %s' % (self._errno, super().__str__())
+
+    @property
+    def errno(self):
+        return self._errno
+
+    @property
+    def message(self):
+        return super().__str__()
+
+
+class NVDeviceAPIError(NVError):
+    pass
+
+
+class NVMLAPIError(NVError):
     pass
 
 
@@ -88,7 +114,7 @@ def _cu_check_error(result):
     if result != CUDA_SUCCESS:
         _error_str = c_char_p()
         _cuda_lib.cuGetErrorString(result, byref(_error_str))
-        raise NVError('Device API Error %d: %s' % (result, _error_str.value.decode()))
+        raise NVDeviceAPIError(_error_str.value.decode(), errno=result)
 
 
 _nvmlErrorString = None
@@ -102,10 +128,7 @@ def _nvml_check_error(result):
 
     if result != NVML_SUCCESS:
         _error_str = _nvmlErrorString(result)
-        if _error_str:
-            raise NVError('NVML API Error %d: %s' % (result, _error_str.decode()))
-        else:
-            raise NVError('Unknown NVML API Error %d' % result)
+        raise NVMLAPIError(_error_str.decode(), errno=result)
 
 
 _cu_process_var_to_cores = {
@@ -123,7 +146,7 @@ def _cu_get_processor_cores(major, minor):
 
 
 def _init_cp():
-    global _cuda_lib
+    global _cuda_lib, _no_device_warned
     if _init_pid == os.getpid():
         return
 
@@ -133,13 +156,19 @@ def _init_cp():
         return
     try:
         _cu_check_error(_cuda_lib.cuInit(0))
-    except NVError:
-        logger.exception('Failed to initialize libcuda.')
+    except NVDeviceAPIError as ex:
+        if ex.errno == CU_NO_CUDA_CAPABLE_DEVICE_DETECTED:
+            _cuda_lib = None
+            if not _no_device_warned:
+                logger.warning('No CUDA device detected')
+                _no_device_warned = True
+        else:
+            logger.exception('Failed to initialize libcuda.')
         return
 
 
 def _init_nvml():
-    global _nvml_lib
+    global _nvml_lib, _no_device_warned
     if _init_pid == os.getpid():
         return
 
@@ -149,8 +178,14 @@ def _init_nvml():
         return
     try:
         _nvml_check_error(_nvml_lib.nvmlInit_v2())
-    except NVError:
-        logger.exception('Failed to initialize libnvidia-ml.')
+    except NVMLAPIError as ex:
+        if ex.errno == NVML_DRIVER_NOT_LOADED:
+            _nvml_lib = None
+            if not _no_device_warned:
+                logger.warning('Failed to load libnvidia-ml: %s, no CUDA device will be enabled', ex.message)
+                _no_device_warned = True
+        else:
+            logger.exception('Failed to initialize libnvidia-ml.')
         return
 
 


Test patch: