Repository: dask/distributed

Issue:
NVML monitoring fails on Window Subsystem for Linux w/ GPU support
<!-- Please include a self-contained copy-pastable example that generates the issue if possible.

Please be concise with code posted. See guidelines below on how to provide a good bug report:

- Craft Minimal Bug Reports http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports
- Minimal Complete Verifiable Examples https://stackoverflow.com/help/mcve

Bug reports that follow these guidelines are easier to diagnose, and so are often handled much more quickly.
-->

**What happened**:
When using an Insider build of Windows Subsystem for Linux (WSL) with GPUs, one quirk is that while GPU metrics are being actively monitored, compute utilization is not made available:

```
 → nvidia-smi
Wed Sep 22 17:41:09 2021
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 510.00       Driver Version: 510.10       CUDA Version: 11.6     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  Quadro RTX 8000     On   | 00000000:15:00.0 Off |                  Off |
| 34%   36C    P8    18W / 260W |    444MiB / 49152MiB |     N/A      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+
|   1  Quadro RTX 8000     On   | 00000000:2D:00.0  On |                  Off |
| 36%   61C    P0    70W / 260W |   2492MiB / 49152MiB |     N/A      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|  No running processes found                                                 |
+-----------------------------------------------------------------------------+
```

Because of this, when trying to start a Distributed cluster on WSL, we immediately see a `NVMLError_NotSupported` as the worker's `SystemMonitors` attempt to fetch the utilization:

```python
In [2]: from distributed import Client

In [3]: client = Client()
---------------------------------------------------------------------------
NVMLError_NotSupported                    Traceback (most recent call last)
<ipython-input-3-84f8000c6f67> in <module>
----> 1 client = Client()

~/distributed/distributed/client.py in __init__(self, address, loop, timeout, set_as_default, scheduler_file, security, asynchronous, name, heartbeat_interval, serializers, deserializers, extensions, direct_to_workers, connection_limit, **kwargs)
    756             ext(self)
    757 
--> 758         self.start(timeout=timeout)
    759         Client._instances.add(self)
    760 

~/distributed/distributed/client.py in start(self, **kwargs)
    938             self._started = asyncio.ensure_future(self._start(**kwargs))
    939         else:
--> 940             sync(self.loop, self._start, **kwargs)
    941 
    942     def __await__(self):

~/distributed/distributed/utils.py in sync(loop, func, callback_timeout, *args, **kwargs)
    324     if error[0]:
    325         typ, exc, tb = error[0]
--> 326         raise exc.with_traceback(tb)
    327     else:
    328         return result[0]

~/distributed/distributed/utils.py in f()
    307             if callback_timeout is not None:
    308                 future = asyncio.wait_for(future, callback_timeout)
--> 309             result[0] = yield future
    310         except Exception:
    311             error[0] = sys.exc_info()

~/compose/etc/conda/cuda_11.2/envs/rapids/lib/python3.8/site-packages/tornado/gen.py in run(self)
    760 
    761                     try:
--> 762                         value = future.result()
    763                     except Exception:
    764                         exc_info = sys.exc_info()

~/distributed/distributed/client.py in _start(self, timeout, **kwargs)
   1003 
   1004             try:
-> 1005                 self.cluster = await LocalCluster(
   1006                     loop=self.loop,
   1007                     asynchronous=self._asynchronous,

~/distributed/distributed/deploy/spec.py in _()
    398         async def _():
    399             if self.status == Status.created:
--> 400                 await self._start()
    401             await self.scheduler
    402             await self._correct_state()

~/distributed/distributed/deploy/spec.py in _start(self)
    309             if isinstance(cls, str):
    310                 cls = import_term(cls)
--> 311             self.scheduler = cls(**self.scheduler_spec.get("options", {}))
    312             self.scheduler = await self.scheduler
    313         self.scheduler_comm = rpc(

~/distributed/distributed/scheduler.py in __init__(self, loop, delete_interval, synchronize_worker_interval, services, service_kwargs, allowed_failures, extensions, validate, scheduler_file, security, worker_ttl, idle_timeout, interface, host, port, protocol, dashboard_address, dashboard, http_prefix, preload, preload_argv, plugins, **kwargs)
   3835         connection_limit = get_fileno_limit() / 2
   3836 
-> 3837         super().__init__(
   3838             aliases=aliases,
   3839             handlers=self.handlers,

~/distributed/distributed/scheduler.py in __init__(self, aliases, clients, workers, host_info, resources, tasks, unrunnable, validate, **kwargs)
   1982         self._transition_counter = 0
   1983 
-> 1984         super().__init__(**kwargs)
   1985 
   1986     @property

~/distributed/distributed/core.py in __init__(self, handlers, blocked_handlers, stream_handlers, connection_limit, deserialize, serializers, deserializers, connection_args, timeout, io_loop)
    158         self._comms = {}
    159         self.deserialize = deserialize
--> 160         self.monitor = SystemMonitor()
    161         self.counters = None
    162         self.digests = None

~/distributed/distributed/system_monitor.py in __init__(self, n)
     65             self.quantities["gpu_memory_used"] = self.gpu_memory_used
     66 
---> 67         self.update()
     68 
     69     def recent(self):

~/distributed/distributed/system_monitor.py in update(self)
    130 
    131         if nvml.device_get_count() > 0:
--> 132             gpu_metrics = nvml.real_time()
    133             self.gpu_utilization.append(gpu_metrics["utilization"])
    134             self.gpu_memory_used.append(gpu_metrics["memory-used"])

~/distributed/distributed/diagnostics/nvml.py in real_time()
     85     h = _pynvml_handles()
     86     return {
---> 87         "utilization": pynvml.nvmlDeviceGetUtilizationRates(h).gpu,
     88         "memory-used": pynvml.nvmlDeviceGetMemoryInfo(h).used,
     89     }

~/compose/etc/conda/cuda_11.2/envs/rapids/lib/python3.8/site-packages/pynvml/nvml.py in nvmlDeviceGetUtilizationRates(handle)
   2056     fn = _nvmlGetFunctionPointer("nvmlDeviceGetUtilizationRates")
   2057     ret = fn(handle, byref(c_util))
-> 2058     _nvmlCheckReturn(ret)
   2059     return c_util
   2060 

~/compose/etc/conda/cuda_11.2/envs/rapids/lib/python3.8/site-packages/pynvml/nvml.py in _nvmlCheckReturn(ret)
    741 def _nvmlCheckReturn(ret):
    742     if (ret != NVML_SUCCESS):
--> 743         raise NVMLError(ret)
    744     return ret
    745 

NVMLError_NotSupported: Not Supported
```

**What you expected to happen**:
It would be nice if, after getting the device handles for the GPUs, we did a check that all desired metrics can be fetched for them. Depending on this check, we could either disable NVML's monitoring altogether, or disable monitoring of the specific metrics that aren't supported.

**Anything else we need to know?**:
Since this issue is occurring on a Windows Insider build, there's no guarantee that it will persist through future builds.

**Environment**:

- Dask version: latest `main`
- Python version: 3.8
- Operating System: Ubuntu 20.04 (WSL2)
- Install method: source



Discussion:

Resolving patch:
diff --git a/distributed/diagnostics/nvml.py b/distributed/diagnostics/nvml.py
--- a/distributed/diagnostics/nvml.py
+++ b/distributed/diagnostics/nvml.py
@@ -81,17 +81,45 @@ def has_cuda_context():
     return False
 
 
+def _get_utilization(h):
+    try:
+        return pynvml.nvmlDeviceGetUtilizationRates(h).gpu
+    except pynvml.NVMLError_NotSupported:
+        return None
+
+
+def _get_memory_used(h):
+    try:
+        return pynvml.nvmlDeviceGetMemoryInfo(h).used
+    except pynvml.NVMLError_NotSupported:
+        return None
+
+
+def _get_memory_total(h):
+    try:
+        return pynvml.nvmlDeviceGetMemoryInfo(h).total
+    except pynvml.NVMLError_NotSupported:
+        return None
+
+
+def _get_name(h):
+    try:
+        return pynvml.nvmlDeviceGetName(h).decode()
+    except pynvml.NVMLError_NotSupported:
+        return None
+
+
 def real_time():
     h = _pynvml_handles()
     return {
-        "utilization": pynvml.nvmlDeviceGetUtilizationRates(h).gpu,
-        "memory-used": pynvml.nvmlDeviceGetMemoryInfo(h).used,
+        "utilization": _get_utilization(h),
+        "memory-used": _get_memory_used(h),
     }
 
 
 def one_time():
     h = _pynvml_handles()
     return {
-        "memory-total": pynvml.nvmlDeviceGetMemoryInfo(h).total,
-        "name": pynvml.nvmlDeviceGetName(h).decode(),
+        "memory-total": _get_memory_total(h),
+        "name": _get_name(h),
     }


Test patch: