Repository: pytorch/ignite

Issue:
gpu_info crashes because it cannot parse "N/A"
## 🐛 Bug description

When trying to use gpu_info, it throws:

```
 File "/home/blackhc/anaconda3/envs/hello-mnist/lib/python3.7/site-packages/ignite/contrib/metrics/gpu_info.py", line 91, in completed
    engine.state.metrics[util_name] = int(util_report['gpu_util'])
ValueError: invalid literal for int() with base 10: 'N/A'
```

There is error handling code above it, but it does not catch the issue ("N/A" is returned).

I assume my GPU does not support it. However, it would be neat to have a graceful failure mode.

Thank you!
Andreas

## Environment

torch 1.5 on a GTX 780 TI (source)
ignite 0.3.0 (conda)
pynvml 8.0.4 (pip)



Discussion:
Hi Andreas, thanks for catching that !

Maybe, we can cast it to -1 or 0 if no support of `gpu_util`. 
Actually, what shows `nvidia-smi` on your GPU ?

Resolving patch:
diff --git a/ignite/contrib/metrics/gpu_info.py b/ignite/contrib/metrics/gpu_info.py
--- a/ignite/contrib/metrics/gpu_info.py
+++ b/ignite/contrib/metrics/gpu_info.py
@@ -11,6 +11,10 @@ class GpuInfo(Metric):
     """Provides GPU information: a) used memory percentage, b) gpu utilization percentage values as Metric
     on each iterations.
 
+    .. Note ::
+
+        In case if gpu utilization reports "N/A" on a given GPU, corresponding metric value is not set.
+
     Examples:
 
         .. code-block:: python
@@ -90,11 +94,14 @@ def completed(self, engine, name):
             util_report = data_by_rank["utilization"]
             if not ("gpu_util" in util_report):
                 warnings.warn(
-                    "GPU utilization information does not provide 'gpu_util' information in " "{}".format(util_report)
+                    "GPU utilization information does not provide 'gpu_util' information in {}".format(util_report)
                 )
                 continue
-
-            engine.state.metrics[util_name] = int(util_report["gpu_util"])
+            try:
+                engine.state.metrics[util_name] = int(util_report["gpu_util"])
+            except ValueError:
+                # Do not set GPU utilization information
+                pass
 
     def attach(self, engine, name="gpu", event_name=Events.ITERATION_COMPLETED):
         engine.add_event_handler(event_name, self.completed, name)


Test patch:
diff --git a/tests/ignite/contrib/metrics/test_gpu_info.py b/tests/ignite/contrib/metrics/test_gpu_info.py
--- a/tests/ignite/contrib/metrics/test_gpu_info.py
+++ b/tests/ignite/contrib/metrics/test_gpu_info.py
@@ -66,13 +66,16 @@ def _test_gpu_info(device="cpu"):
     gpu_info.completed(engine, name="gpu")
 
     assert "gpu:0 mem(%)" in engine.state.metrics
-    assert "gpu:0 util(%)" in engine.state.metrics
 
     assert isinstance(engine.state.metrics["gpu:0 mem(%)"], int)
     assert int(mem_report["used"] * 100.0 / mem_report["total"]) == engine.state.metrics["gpu:0 mem(%)"]
 
-    assert isinstance(engine.state.metrics["gpu:0 util(%)"], int)
-    assert int(util_report["gpu_util"]) == engine.state.metrics["gpu:0 util(%)"]
+    if util_report["gpu_util"] != "N/A":
+        assert "gpu:0 util(%)" in engine.state.metrics
+        assert isinstance(engine.state.metrics["gpu:0 util(%)"], int)
+        assert int(util_report["gpu_util"]) == engine.state.metrics["gpu:0 util(%)"]
+    else:
+        assert "gpu:0 util(%)" not in engine.state.metrics
 
 
 @pytest.mark.skipif(python_below_36 or not (torch.cuda.is_available()), reason="No pynvml for python < 3.6 and no GPU")
@@ -80,6 +83,9 @@ def test_gpu_info_on_cuda():
     _test_gpu_info(device="cuda")
 
 
+query_resp = None
+
+
 @pytest.fixture
 def mock_pynvml_module():
 
@@ -95,7 +101,7 @@ def mock_pynvml_module():
         from pynvml.smi import nvidia_smi
 
         def query(*args, **kwargs):
-            return {"gpu": [{"fb_memory_usage": {"used": 100.0, "total": 11000.0}, "utilization": {"gpu_util": 50.0}}]}
+            return query_resp
 
         def getInstance():
             nvsmi = Mock()
@@ -116,10 +122,17 @@ def mock_gpu_is_available():
 
 @pytest.mark.skipif(torch.cuda.is_available(), reason="No need to mock if has GPU")
 def test_gpu_info_mock(mock_pynvml_module, mock_gpu_is_available):
+    global query_resp
+
+    query_resp = {"gpu": [{"fb_memory_usage": {"used": 100.0, "total": 11000.0}, "utilization": {"gpu_util": 50.0}}]}
 
     assert torch.cuda.is_available()
     _test_gpu_info()
 
+    # Tests https://github.com/pytorch/ignite/issues/1040
+    query_resp = {"gpu": [{"fb_memory_usage": {"used": 100.0, "total": 11000.0}, "utilization": {"gpu_util": "N/A"}}]}
+    _test_gpu_info()
+
     def _test_with_custom_query(resp, warn_msg, check_compute=False):
         from pynvml.smi import nvidia_smi