Repository: easybuilders/easybuild-framework

Issue:
Report available GPUs and NVIDIA driver version in PR test reports
When looking at test reports it would be useful to know what GPUs were available during the build and what NVIDIA driver version is installed.

This would (probably) only be needed when testing software with a CUDA dependency.

Also, the equivalent for non-NVIDIA GPUs.


Discussion:
I like this idea. I would like to also add the GPU and NVIDIA driver version information to `eb --show-system-info`.

I think as a start for the NVIDIA GPUs we could use the properties `gpu_name` and `driver_version`:
```
$ nvidia-smi --query-gpu=gpu_name,driver_version --format=csv
name, driver_version
A100-SXM4-40GB, 460.32.03
A100-SXM4-40GB, 460.32.03
A100-SXM4-40GB, 460.32.03
A100-SXM4-40GB, 460.32.03
```

A test report with GPU information could then look maybe like this:

```
Test report by @sebastianachilles
**SUCCESS**
Build succeeded for 1 out of 1 (1 easyconfigs in total)
jrc0437 - Linux centos linux 8.3.2011, x86_64, AMD EPYC 7742 64-Core Processor (zen2), 4x A100-SXM4-40GB (Driver Version: 460.32.03), Python 3.6.8
See ... for a full test report.
```

Is there any need to restrict it to when you are using a CUDA dep? If the command is available, query it, if you get a non-zero exit code then reporting that is not going to do any harm.
> Is there any need to restrict it to when you are using a CUDA dep? If the command is available, query it, if you get a non-zero exit code then reporting that is not going to do any harm.

Probably not. And it might be helpful when debugging failed builds to know there was a GPU there and the software decided to build for it even on a CPU-only toolchain.

Resolving patch:
diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py
--- a/easybuild/tools/options.py
+++ b/easybuild/tools/options.py
@@ -99,7 +99,7 @@
 from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME
 from easybuild.tools.repository.repository import avail_repositories
 from easybuild.tools.systemtools import UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family
-from easybuild.tools.systemtools import get_cpu_features, get_system_info
+from easybuild.tools.systemtools import get_cpu_features, get_gpu_info, get_system_info
 from easybuild.tools.version import this_is_easybuild
 
 
@@ -1292,6 +1292,7 @@ def show_system_info(self):
         """Show system information."""
         system_info = get_system_info()
         cpu_features = get_cpu_features()
+        gpu_info = get_gpu_info()
         cpu_arch_name = system_info['cpu_arch_name']
         lines = [
             "System information (%s):" % system_info['hostname'],
@@ -1317,6 +1318,19 @@ def show_system_info(self):
             "  -> speed: %s" % system_info['cpu_speed'],
             "  -> cores: %s" % system_info['core_count'],
             "  -> features: %s" % ','.join(cpu_features),
+        ])
+
+        if gpu_info:
+            lines.extend([
+                '',
+                "* GPU:",
+            ])
+            for vendor in gpu_info:
+                lines.append("  -> %s" % vendor)
+                for gpu, num in gpu_info[vendor].items():
+                    lines.append("    -> %sx %s" % (num, gpu))
+
+        lines.extend([
             '',
             "* software:",
             "  -> glibc version: %s" % system_info['glibc_version'],
diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py
--- a/easybuild/tools/systemtools.py
+++ b/easybuild/tools/systemtools.py
@@ -574,6 +574,34 @@ def get_cpu_features():
     return cpu_feat
 
 
+def get_gpu_info():
+    """
+    Get the GPU info
+    """
+    gpu_info = {}
+    os_type = get_os_type()
+
+    if os_type == LINUX:
+        try:
+            cmd = "nvidia-smi --query-gpu=gpu_name,driver_version --format=csv,noheader"
+            _log.debug("Trying to determine NVIDIA GPU info on Linux via cmd '%s'", cmd)
+            out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False)
+            if ec == 0:
+                for line in out.strip().split('\n'):
+                    nvidia_gpu_info = gpu_info.setdefault('NVIDIA', {})
+                    nvidia_gpu_info.setdefault(line, 0)
+                    nvidia_gpu_info[line] += 1
+            else:
+                _log.debug("None zero exit (%s) from nvidia-smi: %s", ec, out)
+        except Exception as err:
+            _log.debug("Exception was raised when running nvidia-smi: %s", err)
+            _log.info("No NVIDIA GPUs detected")
+    else:
+        _log.info("Only know how to get GPU info on Linux, assuming no GPUs are present")
+
+    return gpu_info
+
+
 def get_kernel_name():
     """NO LONGER SUPPORTED: use get_os_type() instead"""
     _log.nosupport("get_kernel_name() is replaced by get_os_type()", '2.0')


Test patch:
diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py
--- a/easybuild/tools/testing.py
+++ b/easybuild/tools/testing.py
@@ -50,7 +50,7 @@
 from easybuild.tools.jenkins import aggregate_xml_in_dirs
 from easybuild.tools.parallelbuild import build_easyconfigs_in_parallel
 from easybuild.tools.robot import resolve_dependencies
-from easybuild.tools.systemtools import UNKNOWN, get_system_info
+from easybuild.tools.systemtools import UNKNOWN, get_gpu_info, get_system_info
 from easybuild.tools.version import FRAMEWORK_VERSION, EASYBLOCKS_VERSION
 
 
@@ -295,11 +295,20 @@ def post_pr_test_report(pr_nrs, repo_type, test_report, msg, init_session_state,
     if system_info['cpu_arch_name'] != UNKNOWN:
         system_info['cpu_model'] += " (%s)" % system_info['cpu_arch_name']
 
+    # add GPU info, if known
+    gpu_info = get_gpu_info()
+    gpu_str = ""
+    if gpu_info:
+        for vendor in gpu_info:
+            for gpu, num in gpu_info[vendor].items():
+                gpu_str += ", %s x %s %s" % (num, vendor, gpu)
+
     os_info = '%(hostname)s - %(os_type)s %(os_name)s %(os_version)s' % system_info
-    short_system_info = "%(os_info)s, %(cpu_arch)s, %(cpu_model)s, Python %(pyver)s" % {
+    short_system_info = "%(os_info)s, %(cpu_arch)s, %(cpu_model)s%(gpu)s, Python %(pyver)s" % {
         'os_info': os_info,
         'cpu_arch': system_info['cpu_arch'],
         'cpu_model': system_info['cpu_model'],
+        'gpu': gpu_str,
         'pyver': system_info['python_version'].split(' ')[0],
     }