Repository: Project-MONAI/MONAI

Issue:
by default less warnings for the tf32 flags
I'm running the AutoRunner() from monai on 8 gpus, these WARNINGS are overwhelming.   It printed them 16 times (probably from DataAnalyzer() which creates several parallel processes), then another 8 WARNINGS when training starts. 

Can we please disable these warnings?  Or at least show it just one time, and not so many. thank you.

_Originally posted by @myron in https://github.com/Project-MONAI/MONAI/issues/6754#issuecomment-1694581955_
 
https://github.com/Project-MONAI/MONAI/blob/2daabf917a1e6afbc738d22db602f4859672a90d/monai/__init__.py#L83-L85


Discussion:

Resolving patch:
diff --git a/monai/config/deviceconfig.py b/monai/config/deviceconfig.py
--- a/monai/config/deviceconfig.py
+++ b/monai/config/deviceconfig.py
@@ -205,6 +205,8 @@ def get_gpu_info() -> OrderedDict:
         _dict_append(output, "CUDA version", lambda: torch.version.cuda)
     cudnn_ver = torch.backends.cudnn.version()
     _dict_append(output, "cuDNN enabled", lambda: bool(cudnn_ver))
+    _dict_append(output, "NVIDIA_TF32_OVERRIDE", os.environ.get("NVIDIA_TF32_OVERRIDE"))
+    _dict_append(output, "TORCH_ALLOW_TF32_CUBLAS_OVERRIDE", os.environ.get("TORCH_ALLOW_TF32_CUBLAS_OVERRIDE"))
 
     if cudnn_ver:
         _dict_append(output, "cuDNN version", lambda: cudnn_ver)
@@ -215,12 +217,12 @@ def get_gpu_info() -> OrderedDict:
 
     for gpu in range(num_gpus):
         gpu_info = torch.cuda.get_device_properties(gpu)
-        _dict_append(output, f"GPU {gpu} Name", lambda: gpu_info.name)
-        _dict_append(output, f"GPU {gpu} Is integrated", lambda: bool(gpu_info.is_integrated))
-        _dict_append(output, f"GPU {gpu} Is multi GPU board", lambda: bool(gpu_info.is_multi_gpu_board))
-        _dict_append(output, f"GPU {gpu} Multi processor count", lambda: gpu_info.multi_processor_count)
-        _dict_append(output, f"GPU {gpu} Total memory (GB)", lambda: round(gpu_info.total_memory / 1024**3, 1))
-        _dict_append(output, f"GPU {gpu} CUDA capability (maj.min)", lambda: f"{gpu_info.major}.{gpu_info.minor}")
+        _dict_append(output, f"GPU {gpu} Name", gpu_info.name)
+        _dict_append(output, f"GPU {gpu} Is integrated", bool(gpu_info.is_integrated))
+        _dict_append(output, f"GPU {gpu} Is multi GPU board", bool(gpu_info.is_multi_gpu_board))
+        _dict_append(output, f"GPU {gpu} Multi processor count", gpu_info.multi_processor_count)
+        _dict_append(output, f"GPU {gpu} Total memory (GB)", round(gpu_info.total_memory / 1024**3, 1))
+        _dict_append(output, f"GPU {gpu} CUDA capability (maj.min)", f"{gpu_info.major}.{gpu_info.minor}")
 
     return output
 
diff --git a/monai/utils/tf32.py b/monai/utils/tf32.py
--- a/monai/utils/tf32.py
+++ b/monai/utils/tf32.py
@@ -52,7 +52,7 @@ def has_ampere_or_later() -> bool:
 @functools.lru_cache(None)
 def detect_default_tf32() -> bool:
     """
-    Dectect if there is anything that may enable TF32 mode by default.
+    Detect if there is anything that may enable TF32 mode by default.
     If any, show a warning message.
     """
     may_enable_tf32 = False
@@ -70,7 +70,7 @@ def detect_default_tf32() -> bool:
             )
             may_enable_tf32 = True
 
-        override_tf32_env_vars = {"NVIDIA_TF32_OVERRIDE": "1", "TORCH_ALLOW_TF32_CUBLAS_OVERRIDE": "1"}
+        override_tf32_env_vars = {"NVIDIA_TF32_OVERRIDE": "1"}  # TORCH_ALLOW_TF32_CUBLAS_OVERRIDE not checked #6907
         for name, override_val in override_tf32_env_vars.items():
             if os.environ.get(name) == override_val:
                 warnings.warn(


Test patch: